From c615f08bec28a3c8610cd1002731c26975d936f7 Mon Sep 17 00:00:00 2001 From: mik-tf Date: Fri, 18 Oct 2024 10:02:44 -0400 Subject: [PATCH 1/7] changed docker-compose dir to grid-backend, grid-hub to grid-hub-bootstrap --- README.md | 4 ++-- {docker-compose => grid-backend}/devnet/.env | 0 {docker-compose => grid-backend}/devnet/.gitignore | 0 {docker-compose => grid-backend}/devnet/.secrets.env-example | 0 {docker-compose => grid-backend}/devnet/Caddyfile-example | 0 {docker-compose => grid-backend}/devnet/docker-compose.yml | 0 {docker-compose => grid-backend}/devnet/install_grid_bknd.sh | 0 {docker-compose => grid-backend}/devnet/open_logs_tmux.sh | 0 {docker-compose => grid-backend}/devnet/re-sync_processor.sh | 0 {docker-compose => grid-backend}/devnet/setup_db.sh | 0 {docker-compose => grid-backend}/devnet/typesBundle.json | 0 {docker-compose => grid-backend}/mainnet/.env | 0 {docker-compose => grid-backend}/mainnet/.gitignore | 0 {docker-compose => grid-backend}/mainnet/.secrets.env-example | 0 {docker-compose => grid-backend}/mainnet/Caddyfile-example | 0 {docker-compose => grid-backend}/mainnet/docker-compose.yml | 0 {docker-compose => grid-backend}/mainnet/install_grid_bknd.sh | 0 {docker-compose => grid-backend}/mainnet/open_logs_tmux.sh | 0 {docker-compose => grid-backend}/mainnet/re-sync_processor.sh | 0 {docker-compose => grid-backend}/mainnet/resync_indexer.sh | 0 {docker-compose => grid-backend}/mainnet/setup_db.sh | 0 {docker-compose => grid-backend}/mainnet/typesBundle.json | 0 {docker-compose => grid-backend}/qanet/.env | 0 {docker-compose => grid-backend}/qanet/.gitignore | 0 {docker-compose => grid-backend}/qanet/.secrets.env-example | 0 {docker-compose => grid-backend}/qanet/Caddyfile-example | 0 {docker-compose => grid-backend}/qanet/docker-compose.yml | 0 {docker-compose => grid-backend}/qanet/install_grid_bknd.sh | 0 {docker-compose => grid-backend}/qanet/open_logs_tmux.sh | 0 {docker-compose => grid-backend}/qanet/re-sync_processor.sh | 0 {docker-compose => grid-backend}/qanet/setup_db.sh | 0 {docker-compose => grid-backend}/qanet/typesBundle.json | 0 {docker-compose => grid-backend}/readme.md | 0 {docker-compose => grid-backend}/testnet/.env | 0 {docker-compose => grid-backend}/testnet/.gitignore | 0 {docker-compose => grid-backend}/testnet/.secrets.env-example | 0 {docker-compose => grid-backend}/testnet/Caddyfile-example | 0 {docker-compose => grid-backend}/testnet/docker-compose.yml | 0 {docker-compose => grid-backend}/testnet/install_grid_bknd.sh | 0 {docker-compose => grid-backend}/testnet/open_logs_tmux.sh | 0 {docker-compose => grid-backend}/testnet/re-sync_processor.sh | 0 {docker-compose => grid-backend}/testnet/resync_indexer.sh | 0 {docker-compose => grid-backend}/testnet/setup_db.sh | 0 {docker-compose => grid-backend}/testnet/typesBundle.json | 0 {grid-hub => grid-hub-bootstrap}/.env | 0 {grid-hub => grid-hub-bootstrap}/.gitignore | 0 {grid-hub => grid-hub-bootstrap}/Caddyfile | 0 {grid-hub => grid-hub-bootstrap}/bootstrap-kernel-sync.py | 0 {grid-hub => grid-hub-bootstrap}/config-bootstrap.py-example | 0 {grid-hub => grid-hub-bootstrap}/config.py-example | 0 {grid-hub => grid-hub-bootstrap}/docker-compose.yml | 0 {grid-hub => grid-hub-bootstrap}/hub-clone.py | 0 {grid-hub => grid-hub-bootstrap}/incremental.py | 0 {grid-hub => grid-hub-bootstrap}/install-hub.sh | 0 {grid-hub => grid-hub-bootstrap}/open_logs_tmux.sh | 0 {grid-hub => grid-hub-bootstrap}/readme.md | 0 56 files changed, 2 insertions(+), 2 deletions(-) rename {docker-compose => grid-backend}/devnet/.env (100%) rename {docker-compose => grid-backend}/devnet/.gitignore (100%) rename {docker-compose => grid-backend}/devnet/.secrets.env-example (100%) rename {docker-compose => grid-backend}/devnet/Caddyfile-example (100%) rename {docker-compose => grid-backend}/devnet/docker-compose.yml (100%) rename {docker-compose => grid-backend}/devnet/install_grid_bknd.sh (100%) rename {docker-compose => grid-backend}/devnet/open_logs_tmux.sh (100%) rename {docker-compose => grid-backend}/devnet/re-sync_processor.sh (100%) rename {docker-compose => grid-backend}/devnet/setup_db.sh (100%) rename {docker-compose => grid-backend}/devnet/typesBundle.json (100%) rename {docker-compose => grid-backend}/mainnet/.env (100%) rename {docker-compose => grid-backend}/mainnet/.gitignore (100%) rename {docker-compose => grid-backend}/mainnet/.secrets.env-example (100%) rename {docker-compose => grid-backend}/mainnet/Caddyfile-example (100%) rename {docker-compose => grid-backend}/mainnet/docker-compose.yml (100%) rename {docker-compose => grid-backend}/mainnet/install_grid_bknd.sh (100%) rename {docker-compose => grid-backend}/mainnet/open_logs_tmux.sh (100%) rename {docker-compose => grid-backend}/mainnet/re-sync_processor.sh (100%) rename {docker-compose => grid-backend}/mainnet/resync_indexer.sh (100%) rename {docker-compose => grid-backend}/mainnet/setup_db.sh (100%) rename {docker-compose => grid-backend}/mainnet/typesBundle.json (100%) rename {docker-compose => grid-backend}/qanet/.env (100%) rename {docker-compose => grid-backend}/qanet/.gitignore (100%) rename {docker-compose => grid-backend}/qanet/.secrets.env-example (100%) rename {docker-compose => grid-backend}/qanet/Caddyfile-example (100%) rename {docker-compose => grid-backend}/qanet/docker-compose.yml (100%) rename {docker-compose => grid-backend}/qanet/install_grid_bknd.sh (100%) rename {docker-compose => grid-backend}/qanet/open_logs_tmux.sh (100%) rename {docker-compose => grid-backend}/qanet/re-sync_processor.sh (100%) rename {docker-compose => grid-backend}/qanet/setup_db.sh (100%) rename {docker-compose => grid-backend}/qanet/typesBundle.json (100%) rename {docker-compose => grid-backend}/readme.md (100%) rename {docker-compose => grid-backend}/testnet/.env (100%) rename {docker-compose => grid-backend}/testnet/.gitignore (100%) rename {docker-compose => grid-backend}/testnet/.secrets.env-example (100%) rename {docker-compose => grid-backend}/testnet/Caddyfile-example (100%) rename {docker-compose => grid-backend}/testnet/docker-compose.yml (100%) rename {docker-compose => grid-backend}/testnet/install_grid_bknd.sh (100%) rename {docker-compose => grid-backend}/testnet/open_logs_tmux.sh (100%) rename {docker-compose => grid-backend}/testnet/re-sync_processor.sh (100%) rename {docker-compose => grid-backend}/testnet/resync_indexer.sh (100%) rename {docker-compose => grid-backend}/testnet/setup_db.sh (100%) rename {docker-compose => grid-backend}/testnet/typesBundle.json (100%) rename {grid-hub => grid-hub-bootstrap}/.env (100%) rename {grid-hub => grid-hub-bootstrap}/.gitignore (100%) rename {grid-hub => grid-hub-bootstrap}/Caddyfile (100%) rename {grid-hub => grid-hub-bootstrap}/bootstrap-kernel-sync.py (100%) rename {grid-hub => grid-hub-bootstrap}/config-bootstrap.py-example (100%) rename {grid-hub => grid-hub-bootstrap}/config.py-example (100%) rename {grid-hub => grid-hub-bootstrap}/docker-compose.yml (100%) rename {grid-hub => grid-hub-bootstrap}/hub-clone.py (100%) rename {grid-hub => grid-hub-bootstrap}/incremental.py (100%) rename {grid-hub => grid-hub-bootstrap}/install-hub.sh (100%) rename {grid-hub => grid-hub-bootstrap}/open_logs_tmux.sh (100%) rename {grid-hub => grid-hub-bootstrap}/readme.md (100%) diff --git a/README.md b/README.md index 653eb0c..e818f39 100644 --- a/README.md +++ b/README.md @@ -14,7 +14,7 @@ ## Introduction -This repo provides all tools required to set up a Threefold Grid Full Stack. A TFGrid Full Stack is completely stand-alone and made up of several services. It provides all available grid functionalities. +This repo provides all tools required for a Threefold Grid Full Stack deployment. A TFGrid Full Stack is completely stand-alone and made up of several services. It provides all available grid functionalities. ## TFGrid Full Stack Components @@ -29,7 +29,7 @@ The TFGrid Full Stack is composed of: For each of the Threefold Grid networks (Devnet, QAnet, Testnet & Mainnet), the respective grid backend can be deployed using docker compose. -Have a look at the [documentation](https://github.com/threefoldtech/grid_deployment/tree/development/docker-compose) to get started. +Have a look at the [documentation](https://github.com/threefoldtech/grid_deployment/tree/development/grid-backend) to get started. ## TFGrid Zero-OS Hub diff --git a/docker-compose/devnet/.env b/grid-backend/devnet/.env similarity index 100% rename from docker-compose/devnet/.env rename to grid-backend/devnet/.env diff --git a/docker-compose/devnet/.gitignore b/grid-backend/devnet/.gitignore similarity index 100% rename from docker-compose/devnet/.gitignore rename to grid-backend/devnet/.gitignore diff --git a/docker-compose/devnet/.secrets.env-example b/grid-backend/devnet/.secrets.env-example similarity index 100% rename from docker-compose/devnet/.secrets.env-example rename to grid-backend/devnet/.secrets.env-example diff --git a/docker-compose/devnet/Caddyfile-example b/grid-backend/devnet/Caddyfile-example similarity index 100% rename from docker-compose/devnet/Caddyfile-example rename to grid-backend/devnet/Caddyfile-example diff --git a/docker-compose/devnet/docker-compose.yml b/grid-backend/devnet/docker-compose.yml similarity index 100% rename from docker-compose/devnet/docker-compose.yml rename to grid-backend/devnet/docker-compose.yml diff --git a/docker-compose/devnet/install_grid_bknd.sh b/grid-backend/devnet/install_grid_bknd.sh similarity index 100% rename from docker-compose/devnet/install_grid_bknd.sh rename to grid-backend/devnet/install_grid_bknd.sh diff --git a/docker-compose/devnet/open_logs_tmux.sh b/grid-backend/devnet/open_logs_tmux.sh similarity index 100% rename from docker-compose/devnet/open_logs_tmux.sh rename to grid-backend/devnet/open_logs_tmux.sh diff --git a/docker-compose/devnet/re-sync_processor.sh b/grid-backend/devnet/re-sync_processor.sh similarity index 100% rename from docker-compose/devnet/re-sync_processor.sh rename to grid-backend/devnet/re-sync_processor.sh diff --git a/docker-compose/devnet/setup_db.sh b/grid-backend/devnet/setup_db.sh similarity index 100% rename from docker-compose/devnet/setup_db.sh rename to grid-backend/devnet/setup_db.sh diff --git a/docker-compose/devnet/typesBundle.json b/grid-backend/devnet/typesBundle.json similarity index 100% rename from docker-compose/devnet/typesBundle.json rename to grid-backend/devnet/typesBundle.json diff --git a/docker-compose/mainnet/.env b/grid-backend/mainnet/.env similarity index 100% rename from docker-compose/mainnet/.env rename to grid-backend/mainnet/.env diff --git a/docker-compose/mainnet/.gitignore b/grid-backend/mainnet/.gitignore similarity index 100% rename from docker-compose/mainnet/.gitignore rename to grid-backend/mainnet/.gitignore diff --git a/docker-compose/mainnet/.secrets.env-example b/grid-backend/mainnet/.secrets.env-example similarity index 100% rename from docker-compose/mainnet/.secrets.env-example rename to grid-backend/mainnet/.secrets.env-example diff --git a/docker-compose/mainnet/Caddyfile-example b/grid-backend/mainnet/Caddyfile-example similarity index 100% rename from docker-compose/mainnet/Caddyfile-example rename to grid-backend/mainnet/Caddyfile-example diff --git a/docker-compose/mainnet/docker-compose.yml b/grid-backend/mainnet/docker-compose.yml similarity index 100% rename from docker-compose/mainnet/docker-compose.yml rename to grid-backend/mainnet/docker-compose.yml diff --git a/docker-compose/mainnet/install_grid_bknd.sh b/grid-backend/mainnet/install_grid_bknd.sh similarity index 100% rename from docker-compose/mainnet/install_grid_bknd.sh rename to grid-backend/mainnet/install_grid_bknd.sh diff --git a/docker-compose/mainnet/open_logs_tmux.sh b/grid-backend/mainnet/open_logs_tmux.sh similarity index 100% rename from docker-compose/mainnet/open_logs_tmux.sh rename to grid-backend/mainnet/open_logs_tmux.sh diff --git a/docker-compose/mainnet/re-sync_processor.sh b/grid-backend/mainnet/re-sync_processor.sh similarity index 100% rename from docker-compose/mainnet/re-sync_processor.sh rename to grid-backend/mainnet/re-sync_processor.sh diff --git a/docker-compose/mainnet/resync_indexer.sh b/grid-backend/mainnet/resync_indexer.sh similarity index 100% rename from docker-compose/mainnet/resync_indexer.sh rename to grid-backend/mainnet/resync_indexer.sh diff --git a/docker-compose/mainnet/setup_db.sh b/grid-backend/mainnet/setup_db.sh similarity index 100% rename from docker-compose/mainnet/setup_db.sh rename to grid-backend/mainnet/setup_db.sh diff --git a/docker-compose/mainnet/typesBundle.json b/grid-backend/mainnet/typesBundle.json similarity index 100% rename from docker-compose/mainnet/typesBundle.json rename to grid-backend/mainnet/typesBundle.json diff --git a/docker-compose/qanet/.env b/grid-backend/qanet/.env similarity index 100% rename from docker-compose/qanet/.env rename to grid-backend/qanet/.env diff --git a/docker-compose/qanet/.gitignore b/grid-backend/qanet/.gitignore similarity index 100% rename from docker-compose/qanet/.gitignore rename to grid-backend/qanet/.gitignore diff --git a/docker-compose/qanet/.secrets.env-example b/grid-backend/qanet/.secrets.env-example similarity index 100% rename from docker-compose/qanet/.secrets.env-example rename to grid-backend/qanet/.secrets.env-example diff --git a/docker-compose/qanet/Caddyfile-example b/grid-backend/qanet/Caddyfile-example similarity index 100% rename from docker-compose/qanet/Caddyfile-example rename to grid-backend/qanet/Caddyfile-example diff --git a/docker-compose/qanet/docker-compose.yml b/grid-backend/qanet/docker-compose.yml similarity index 100% rename from docker-compose/qanet/docker-compose.yml rename to grid-backend/qanet/docker-compose.yml diff --git a/docker-compose/qanet/install_grid_bknd.sh b/grid-backend/qanet/install_grid_bknd.sh similarity index 100% rename from docker-compose/qanet/install_grid_bknd.sh rename to grid-backend/qanet/install_grid_bknd.sh diff --git a/docker-compose/qanet/open_logs_tmux.sh b/grid-backend/qanet/open_logs_tmux.sh similarity index 100% rename from docker-compose/qanet/open_logs_tmux.sh rename to grid-backend/qanet/open_logs_tmux.sh diff --git a/docker-compose/qanet/re-sync_processor.sh b/grid-backend/qanet/re-sync_processor.sh similarity index 100% rename from docker-compose/qanet/re-sync_processor.sh rename to grid-backend/qanet/re-sync_processor.sh diff --git a/docker-compose/qanet/setup_db.sh b/grid-backend/qanet/setup_db.sh similarity index 100% rename from docker-compose/qanet/setup_db.sh rename to grid-backend/qanet/setup_db.sh diff --git a/docker-compose/qanet/typesBundle.json b/grid-backend/qanet/typesBundle.json similarity index 100% rename from docker-compose/qanet/typesBundle.json rename to grid-backend/qanet/typesBundle.json diff --git a/docker-compose/readme.md b/grid-backend/readme.md similarity index 100% rename from docker-compose/readme.md rename to grid-backend/readme.md diff --git a/docker-compose/testnet/.env b/grid-backend/testnet/.env similarity index 100% rename from docker-compose/testnet/.env rename to grid-backend/testnet/.env diff --git a/docker-compose/testnet/.gitignore b/grid-backend/testnet/.gitignore similarity index 100% rename from docker-compose/testnet/.gitignore rename to grid-backend/testnet/.gitignore diff --git a/docker-compose/testnet/.secrets.env-example b/grid-backend/testnet/.secrets.env-example similarity index 100% rename from docker-compose/testnet/.secrets.env-example rename to grid-backend/testnet/.secrets.env-example diff --git a/docker-compose/testnet/Caddyfile-example b/grid-backend/testnet/Caddyfile-example similarity index 100% rename from docker-compose/testnet/Caddyfile-example rename to grid-backend/testnet/Caddyfile-example diff --git a/docker-compose/testnet/docker-compose.yml b/grid-backend/testnet/docker-compose.yml similarity index 100% rename from docker-compose/testnet/docker-compose.yml rename to grid-backend/testnet/docker-compose.yml diff --git a/docker-compose/testnet/install_grid_bknd.sh b/grid-backend/testnet/install_grid_bknd.sh similarity index 100% rename from docker-compose/testnet/install_grid_bknd.sh rename to grid-backend/testnet/install_grid_bknd.sh diff --git a/docker-compose/testnet/open_logs_tmux.sh b/grid-backend/testnet/open_logs_tmux.sh similarity index 100% rename from docker-compose/testnet/open_logs_tmux.sh rename to grid-backend/testnet/open_logs_tmux.sh diff --git a/docker-compose/testnet/re-sync_processor.sh b/grid-backend/testnet/re-sync_processor.sh similarity index 100% rename from docker-compose/testnet/re-sync_processor.sh rename to grid-backend/testnet/re-sync_processor.sh diff --git a/docker-compose/testnet/resync_indexer.sh b/grid-backend/testnet/resync_indexer.sh similarity index 100% rename from docker-compose/testnet/resync_indexer.sh rename to grid-backend/testnet/resync_indexer.sh diff --git a/docker-compose/testnet/setup_db.sh b/grid-backend/testnet/setup_db.sh similarity index 100% rename from docker-compose/testnet/setup_db.sh rename to grid-backend/testnet/setup_db.sh diff --git a/docker-compose/testnet/typesBundle.json b/grid-backend/testnet/typesBundle.json similarity index 100% rename from docker-compose/testnet/typesBundle.json rename to grid-backend/testnet/typesBundle.json diff --git a/grid-hub/.env b/grid-hub-bootstrap/.env similarity index 100% rename from grid-hub/.env rename to grid-hub-bootstrap/.env diff --git a/grid-hub/.gitignore b/grid-hub-bootstrap/.gitignore similarity index 100% rename from grid-hub/.gitignore rename to grid-hub-bootstrap/.gitignore diff --git a/grid-hub/Caddyfile b/grid-hub-bootstrap/Caddyfile similarity index 100% rename from grid-hub/Caddyfile rename to grid-hub-bootstrap/Caddyfile diff --git a/grid-hub/bootstrap-kernel-sync.py b/grid-hub-bootstrap/bootstrap-kernel-sync.py similarity index 100% rename from grid-hub/bootstrap-kernel-sync.py rename to grid-hub-bootstrap/bootstrap-kernel-sync.py diff --git a/grid-hub/config-bootstrap.py-example b/grid-hub-bootstrap/config-bootstrap.py-example similarity index 100% rename from grid-hub/config-bootstrap.py-example rename to grid-hub-bootstrap/config-bootstrap.py-example diff --git a/grid-hub/config.py-example b/grid-hub-bootstrap/config.py-example similarity index 100% rename from grid-hub/config.py-example rename to grid-hub-bootstrap/config.py-example diff --git a/grid-hub/docker-compose.yml b/grid-hub-bootstrap/docker-compose.yml similarity index 100% rename from grid-hub/docker-compose.yml rename to grid-hub-bootstrap/docker-compose.yml diff --git a/grid-hub/hub-clone.py b/grid-hub-bootstrap/hub-clone.py similarity index 100% rename from grid-hub/hub-clone.py rename to grid-hub-bootstrap/hub-clone.py diff --git a/grid-hub/incremental.py b/grid-hub-bootstrap/incremental.py similarity index 100% rename from grid-hub/incremental.py rename to grid-hub-bootstrap/incremental.py diff --git a/grid-hub/install-hub.sh b/grid-hub-bootstrap/install-hub.sh similarity index 100% rename from grid-hub/install-hub.sh rename to grid-hub-bootstrap/install-hub.sh diff --git a/grid-hub/open_logs_tmux.sh b/grid-hub-bootstrap/open_logs_tmux.sh similarity index 100% rename from grid-hub/open_logs_tmux.sh rename to grid-hub-bootstrap/open_logs_tmux.sh diff --git a/grid-hub/readme.md b/grid-hub-bootstrap/readme.md similarity index 100% rename from grid-hub/readme.md rename to grid-hub-bootstrap/readme.md From 817c6d3da5a471d7131c7251551ae3a8229730dc Mon Sep 17 00:00:00 2001 From: mik-tf Date: Fri, 18 Oct 2024 10:09:18 -0400 Subject: [PATCH 2/7] updated readme for snapshots --- README.md | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index e818f39..dccd5a4 100644 --- a/README.md +++ b/README.md @@ -8,7 +8,7 @@ - [TFGrid Zero-OS Hub](#tfgrid-zero-os-hub) - [TFGrid Zero-OS Bootstrap Generator](#tfgrid-zero-os-bootstrap-generator) - [TFChain Validator](#tfchain-validator) -- [TFGrid Snapshots](#tfgrid-snapshots) +- [TFGrid and TFChain Validator Snapshots](#tfgrid-and-tfchain-validator-snapshots) --- @@ -35,16 +35,26 @@ Have a look at the [documentation](https://github.com/threefoldtech/grid_deploym The Zero-OS Hub is used to distribute Flist files. Users use Flists to deploy workloads on 3Nodes. +Have a look at the [documentation](https://github.com/threefoldtech/grid_deployment/tree/development/grid-hub-bootstrap) to get started. + ## TFGrid Zero-OS Bootstrap Generator The bootstrap service is used to create ZOS bootstrap images. Farmers use bootstrap images to set up 3Nodes on the TFGrid. +Have a look at the [documentation](https://github.com/threefoldtech/grid_deployment/tree/development/grid-hub-bootstrap) to get started. + ## TFChain Validator The TFGrid runs on TFChain. An easy-to-use installer is provided to set up a validator. -## TFGrid Snapshots +Have a look at the [documentation](https://github.com/threefoldtech/grid_deployment/tree/development/tfchain-validator) to get started. + +## TFGrid and TFChain Validator Snapshots + +Daily TFGrid and TFChain validator snapshots can be found at [https://bknd.snapshot.grid.tf/](https://bknd.snapshot.grid.tf/). + +To set up your own TFGrid snapshot creation, read [this documentation](https://github.com/threefoldtech/grid_deployment/tree/development/grid-snapshots). -Daily snapshots can be found [here](https://bknd.snapshot.grid.tf/). To set up your own snapshot creation, read [this documentation](https://github.com/threefoldtech/grid_deployment/tree/development/grid-snapshots). +To set up your own TFChain validator snapshot creation, read [this documentation](https://github.com/threefoldtech/grid_deployment/tree/development/tfchain-validator-snapshots). -You can also use Rsync to download the snapshots. Check [this link](https://github.com/threefoldtech/grid_deployment/tree/development/grid-snapshots#public-rsync-provided-by-threefold) for more information. +> Note: You can also use Rsync to download the snapshots. Read [this section](https://github.com/threefoldtech/grid_deployment/tree/development/grid-snapshots#public-rsync-provided-by-threefold) for more information. From b0594abc952589836431ccf265937dc3464bcccb Mon Sep 17 00:00:00 2001 From: mik-tf Date: Fri, 18 Oct 2024 10:17:02 -0400 Subject: [PATCH 3/7] updated snapshots readme --- README.md | 18 +-- grid-snapshots/readme.md | 2 +- tfchain-validator-snapshots/readme.md | 175 ++++++++++++++++++++++++++ 3 files changed, 185 insertions(+), 10 deletions(-) create mode 100644 tfchain-validator-snapshots/readme.md diff --git a/README.md b/README.md index dccd5a4..a8bd6c1 100644 --- a/README.md +++ b/README.md @@ -14,11 +14,11 @@ ## Introduction -This repo provides all tools required for a Threefold Grid Full Stack deployment. A TFGrid Full Stack is completely stand-alone and made up of several services. It provides all available grid functionalities. +This repo provides all tools required for a Threefold Grid full stack deployment. A TFGrid full stack is completely stand-alone and made up of several services. It provides all available grid functionalities. ## TFGrid Full Stack Components -The TFGrid Full Stack is composed of: +The TFGrid full stack is composed of: - TFGrid Backend - TFGrid Zero-OS Hub @@ -29,32 +29,32 @@ The TFGrid Full Stack is composed of: For each of the Threefold Grid networks (Devnet, QAnet, Testnet & Mainnet), the respective grid backend can be deployed using docker compose. -Have a look at the [documentation](https://github.com/threefoldtech/grid_deployment/tree/development/grid-backend) to get started. +Have a look at the [documentation](./grid-backend/readme.md) to get started. ## TFGrid Zero-OS Hub The Zero-OS Hub is used to distribute Flist files. Users use Flists to deploy workloads on 3Nodes. -Have a look at the [documentation](https://github.com/threefoldtech/grid_deployment/tree/development/grid-hub-bootstrap) to get started. +Have a look at the [documentation](./grid-hub-bootstrap/readme.md) to get started. ## TFGrid Zero-OS Bootstrap Generator The bootstrap service is used to create ZOS bootstrap images. Farmers use bootstrap images to set up 3Nodes on the TFGrid. -Have a look at the [documentation](https://github.com/threefoldtech/grid_deployment/tree/development/grid-hub-bootstrap) to get started. +Have a look at the [documentation](./grid-hub-bootstrap/readme.md) to get started. ## TFChain Validator The TFGrid runs on TFChain. An easy-to-use installer is provided to set up a validator. -Have a look at the [documentation](https://github.com/threefoldtech/grid_deployment/tree/development/tfchain-validator) to get started. +Have a look at the [documentation](./tfchain-validator/readme.md) to get started. ## TFGrid and TFChain Validator Snapshots Daily TFGrid and TFChain validator snapshots can be found at [https://bknd.snapshot.grid.tf/](https://bknd.snapshot.grid.tf/). -To set up your own TFGrid snapshot creation, read [this documentation](https://github.com/threefoldtech/grid_deployment/tree/development/grid-snapshots). +To set up your own TFGrid snapshot creation, read [this documentation](./grid-snapshots/readme.md). -To set up your own TFChain validator snapshot creation, read [this documentation](https://github.com/threefoldtech/grid_deployment/tree/development/tfchain-validator-snapshots). +To set up your own TFChain validator snapshot creation, read [this documentation](./tfchain-validator-snapshots/readme.md). -> Note: You can also use Rsync to download the snapshots. Read [this section](https://github.com/threefoldtech/grid_deployment/tree/development/grid-snapshots#public-rsync-provided-by-threefold) for more information. +> Note: You can also use Rsync to download the snapshots. Read [this section](./grid-snapshots/readme.md#threefold-public-rsync) for more information. diff --git a/grid-snapshots/readme.md b/grid-snapshots/readme.md index 8d30aad..9070410 100644 --- a/grid-snapshots/readme.md +++ b/grid-snapshots/readme.md @@ -1,4 +1,4 @@ -

TFGrid Backend Snapshots

+

TFChain Validator Snapshots

Table of Contents

diff --git a/tfchain-validator-snapshots/readme.md b/tfchain-validator-snapshots/readme.md new file mode 100644 index 0000000..9070410 --- /dev/null +++ b/tfchain-validator-snapshots/readme.md @@ -0,0 +1,175 @@ +

TFChain Validator Snapshots

+ +

Table of Contents

+ +- [Introduction](#introduction) +- [ThreeFold Public Rsync](#threefold-public-rsync) +- [Services](#services) +- [Requirements](#requirements) + - [Configuration](#configuration) + - [Hardware](#hardware) + - [Files for Each Net](#files-for-each-net) +- [Deployment](#deployment) +- [Script](#script) +- [Rsync](#rsync) + +--- + +## Introduction + +To facilitate the deployment of grid backend services, we provide snapshots to significantly reduce sync time. This can be setup anywhere from scratch. Once all services are synced, one can use the scripts to create snapshots automatically. + +## ThreeFold Public Rsync + +Threefold hosts all available snapshots at: [https://bknd.snapshot.grid.tf/](https://bknd.snapshot.grid.tf/), which can be downloaded with rsync as shown below: + +- Mainnet: + ``` + rsync -Lv --progress --partial rsync://bknd.snapshot.grid.tf:34873/gridsnapshots/tfchain-mainnet-latest.tar.gz . + rsync -Lv --progress --partial rsync://bknd.snapshot.grid.tf:34873/gridsnapshots/indexer-mainnet-latest.tar.gz . + rsync -Lv --progress --partial rsync://bknd.snapshot.grid.tf:34873/gridsnapshots/processor-mainnet-latest.tar.gz . + ``` + +- Testnet: + ``` + rsync -Lv --progress --partial rsync://bknd.snapshot.grid.tf:34873/gridsnapshotstest/tfchain-testnet-latest.tar.gz . + rsync -Lv --progress --partial rsync://bknd.snapshot.grid.tf:34873/gridsnapshotstest/indexer-testnet-latest.tar.gz . + rsync -Lv --progress --partial rsync://bknd.snapshot.grid.tf:34873/gridsnapshotstest/processor-testnet-latest.tar.gz . + ``` +- Devnet: + ``` + rsync -Lv --progress --partial rsync://bknd.snapshot.grid.tf:34873/gridsnapshotsdev/tfchain-devnet-latest.tar.gz . + rsync -Lv --progress --partial rsync://bknd.snapshot.grid.tf:34873/gridsnapshotsdev/indexer-devnet-latest.tar.gz . + rsync -Lv --progress --partial rsync://bknd.snapshot.grid.tf:34873/gridsnapshotsdev/processor-devnet-latest.tar.gz . + ``` +## Services + +There are 3 TFGrid backend services that collect enough data to justify creating snapshots: + +- Threefold blockchain - TFchain +- Graphql - Indexer +- Graphql - Processor + + +## Requirements + +To run your own snapshot backend one needs the following + +### Configuration + +- A working docker environment +- 'node key' for the TFchain public RPC node, generated with `subkey generate-node-key` + +### Hardware + +- Min of 8 modern CPU cores +- Min of 32GB RAM +- Min of 1TB SSD storage (high preference for NVMe based storage) preferably more (as the chain keeps growing in size) +- Min of 2TB HDD storage (to store and share the snapshots) + +Dev, QA and Testnet can do with a Sata SSD setup. Mainnet requires NVMe based SSDs due to the data size. + +> Note: If a deployment does not have enough disk iops available one can see the processor container restarting regulary alongside grid_proxy errors regarding processor database timeouts. + + +### Files for Each Net + +Each folder contains the required deployment files for it's net, work in the folder that has the name of the network you want to create snapshots for. + +What does each file do: +- `.env` - contains environment files maintaned by Threefold Tech +- `.gitignore` - has a list of files to ignore once the repo has been cloned. This has the purpose to not have uncommited changes to files when working in this repo +- `.secrets.env-examples` - is where you have to add all your unique environment variables +- `create_snapshot.sh` - script to create a snapshot (used by cron) +- `docker-compose.yml` - has all the required docker-compose configuration to deploy a working Grid stack +- `open_logs_tmux.sh` - opens all the docker logs in tmux sessions +- `typesBundle.json` - contains data for the Graphql indexer and is not to be touched +- `startall.sh` - starts all the (already deployed) containers +- `stopall.sh` - stops all the (already deployed) containers + + +## Deployment + +`cd` into the correct folder for the network your deploying for, our example uses mainnet. + +```sh +cd mainnet +cp .secrets.env-example .secrets.env +``` + +Open `.secrets.env` and add your generated subkey node-key + +Check if all environment variables are correct. +``` +docker compose --env-file .secrets.env --env-file .env config +``` + +Deploy the snapshot backend. Depending on the disk iops available, it can take up until a week to sync from block 0. + +```sh +docker compose --env-file .secrets.env --env-file .env up -d +``` + +## Script + +Add the appropriate script to cron with an interval you want the snapshots to be created + +```sh +mkdir /var/log/snapshots +touch /var/log/snapshots/snapshots-cron.log +crontab -e +``` + +```sh +0 1 * * * sh /root/code/grid_deployment/grid-snapshots/mainnet/create_snapshot.sh > /var/log/snapshots/snapshots-cron.log 2>&1 +``` + +This example will execute the script every day at 1 AM and send the logs to /var/log/snapshots/snapshots-cron.log + + +## Rsync + +We use rsync to expose the snapshots to the community. To setup a public rsync server create and edit the following file: + +- File `/etc/rsyncd.conf`: + +```sh +pid file = /var/run/rsyncd.pid +lock file = /var/run/rsync.lock +log file = /var/log/rsync.log +port = 34873 +max connections = 20 +exclude = lost+found/ +transfer logging = yes +use chroot = yes +reverse lookup = no + +[gridsnapshots] +path = /storage/rsync-public/mainnet +comment = THREEFOLD GRID MAINNET SNAPSHOTS +read only = true +timeout = 300 +list = false + +[gridsnapshotstest] +path = /storage/rsync-public/testnet +comment = THREEFOLD GRID TESTNET SNAPSHOTS +read only = true +timeout = 300 +list = false + +[gridsnapshotsdev] +path = /storage/rsync-public/devnet +comment = THREEFOLD GRID DEVNET SNAPSHOTS +read only = true +timeout = 300 +list = false +``` + +Start and enable the service with systemd: + +```sh +systemctl start rsync +systemctl enable rsync +systemctl status rsync +``` From 9f7b7a8f9196dcaa50b1e569e9843df8e9fcbb7b Mon Sep 17 00:00:00 2001 From: mik-tf Date: Fri, 18 Oct 2024 10:18:49 -0400 Subject: [PATCH 4/7] fixed typo in snapshots gridbackend --- grid-snapshots/readme.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/grid-snapshots/readme.md b/grid-snapshots/readme.md index 9070410..8d30aad 100644 --- a/grid-snapshots/readme.md +++ b/grid-snapshots/readme.md @@ -1,4 +1,4 @@ -

TFChain Validator Snapshots

+

TFGrid Backend Snapshots

Table of Contents

From d239a7b33ff74397ebbbd0b058a20e58fd096efb Mon Sep 17 00:00:00 2001 From: mik-tf Date: Fri, 18 Oct 2024 10:24:22 -0400 Subject: [PATCH 5/7] optimized readme for snapshots steps --- README.md | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index a8bd6c1..6662abc 100644 --- a/README.md +++ b/README.md @@ -53,8 +53,6 @@ Have a look at the [documentation](./tfchain-validator/readme.md) to get started Daily TFGrid and TFChain validator snapshots can be found at [https://bknd.snapshot.grid.tf/](https://bknd.snapshot.grid.tf/). -To set up your own TFGrid snapshot creation, read [this documentation](./grid-snapshots/readme.md). +To set up your own TFGrid and TFChain validator snapshot creation, read [this documentation](./grid-snapshots/readme.md). The snapshot deployment process sets up both the TFGrid snapshots and TFChain validator snapshots at once. -To set up your own TFChain validator snapshot creation, read [this documentation](./tfchain-validator-snapshots/readme.md). - -> Note: You can also use Rsync to download the snapshots. Read [this section](./grid-snapshots/readme.md#threefold-public-rsync) for more information. +> Note: You can also use Rsync to download the snapshots. Read [this section](./grid-snapshots/readme.md#threefold-public-rsync) for more information. \ No newline at end of file From 24bc44b347513a465fa647b8af5beb8987d5ddb5 Mon Sep 17 00:00:00 2001 From: mik-tf Date: Fri, 18 Oct 2024 10:32:03 -0400 Subject: [PATCH 6/7] set snapshot readme preparation for validator --- README.md | 4 +- grid-snapshots/readme.md | 2 +- tfchain-validator-snapshots/readme.md | 167 +------------------------- 3 files changed, 8 insertions(+), 165 deletions(-) diff --git a/README.md b/README.md index 6662abc..0344cb6 100644 --- a/README.md +++ b/README.md @@ -53,6 +53,8 @@ Have a look at the [documentation](./tfchain-validator/readme.md) to get started Daily TFGrid and TFChain validator snapshots can be found at [https://bknd.snapshot.grid.tf/](https://bknd.snapshot.grid.tf/). -To set up your own TFGrid and TFChain validator snapshot creation, read [this documentation](./grid-snapshots/readme.md). The snapshot deployment process sets up both the TFGrid snapshots and TFChain validator snapshots at once. +To set up your own TFGrid snapshot creation, read [this documentation](./grid-snapshots/readme.md). + +To set up your own TFChain validator snapshot creation, read [this documentation](./tfchain-validator-snapshots/readme.md). > Note: You can also use Rsync to download the snapshots. Read [this section](./grid-snapshots/readme.md#threefold-public-rsync) for more information. \ No newline at end of file diff --git a/grid-snapshots/readme.md b/grid-snapshots/readme.md index 8d30aad..f785326 100644 --- a/grid-snapshots/readme.md +++ b/grid-snapshots/readme.md @@ -17,7 +17,7 @@ ## Introduction -To facilitate the deployment of grid backend services, we provide snapshots to significantly reduce sync time. This can be setup anywhere from scratch. Once all services are synced, one can use the scripts to create snapshots automatically. +To facilitate the deployment of grid backend services, we provide TFGrid backend snapshots to significantly reduce sync time. This can be setup anywhere from scratch. Once all services are synced, one can use the scripts to create snapshots automatically. ## ThreeFold Public Rsync diff --git a/tfchain-validator-snapshots/readme.md b/tfchain-validator-snapshots/readme.md index 9070410..ec796d0 100644 --- a/tfchain-validator-snapshots/readme.md +++ b/tfchain-validator-snapshots/readme.md @@ -3,173 +3,14 @@

Table of Contents

- [Introduction](#introduction) -- [ThreeFold Public Rsync](#threefold-public-rsync) -- [Services](#services) -- [Requirements](#requirements) - - [Configuration](#configuration) - - [Hardware](#hardware) - - [Files for Each Net](#files-for-each-net) -- [Deployment](#deployment) -- [Script](#script) -- [Rsync](#rsync) +- [TODO](#todo) --- ## Introduction -To facilitate the deployment of grid backend services, we provide snapshots to significantly reduce sync time. This can be setup anywhere from scratch. Once all services are synced, one can use the scripts to create snapshots automatically. +To facilitate the deployment of grid backend services, we provide TFChain validator snapshots to significantly reduce sync time. This can be setup anywhere from scratch. Once all services are synced, one can use the scripts to create snapshots automatically. -## ThreeFold Public Rsync +## TODO -Threefold hosts all available snapshots at: [https://bknd.snapshot.grid.tf/](https://bknd.snapshot.grid.tf/), which can be downloaded with rsync as shown below: - -- Mainnet: - ``` - rsync -Lv --progress --partial rsync://bknd.snapshot.grid.tf:34873/gridsnapshots/tfchain-mainnet-latest.tar.gz . - rsync -Lv --progress --partial rsync://bknd.snapshot.grid.tf:34873/gridsnapshots/indexer-mainnet-latest.tar.gz . - rsync -Lv --progress --partial rsync://bknd.snapshot.grid.tf:34873/gridsnapshots/processor-mainnet-latest.tar.gz . - ``` - -- Testnet: - ``` - rsync -Lv --progress --partial rsync://bknd.snapshot.grid.tf:34873/gridsnapshotstest/tfchain-testnet-latest.tar.gz . - rsync -Lv --progress --partial rsync://bknd.snapshot.grid.tf:34873/gridsnapshotstest/indexer-testnet-latest.tar.gz . - rsync -Lv --progress --partial rsync://bknd.snapshot.grid.tf:34873/gridsnapshotstest/processor-testnet-latest.tar.gz . - ``` -- Devnet: - ``` - rsync -Lv --progress --partial rsync://bknd.snapshot.grid.tf:34873/gridsnapshotsdev/tfchain-devnet-latest.tar.gz . - rsync -Lv --progress --partial rsync://bknd.snapshot.grid.tf:34873/gridsnapshotsdev/indexer-devnet-latest.tar.gz . - rsync -Lv --progress --partial rsync://bknd.snapshot.grid.tf:34873/gridsnapshotsdev/processor-devnet-latest.tar.gz . - ``` -## Services - -There are 3 TFGrid backend services that collect enough data to justify creating snapshots: - -- Threefold blockchain - TFchain -- Graphql - Indexer -- Graphql - Processor - - -## Requirements - -To run your own snapshot backend one needs the following - -### Configuration - -- A working docker environment -- 'node key' for the TFchain public RPC node, generated with `subkey generate-node-key` - -### Hardware - -- Min of 8 modern CPU cores -- Min of 32GB RAM -- Min of 1TB SSD storage (high preference for NVMe based storage) preferably more (as the chain keeps growing in size) -- Min of 2TB HDD storage (to store and share the snapshots) - -Dev, QA and Testnet can do with a Sata SSD setup. Mainnet requires NVMe based SSDs due to the data size. - -> Note: If a deployment does not have enough disk iops available one can see the processor container restarting regulary alongside grid_proxy errors regarding processor database timeouts. - - -### Files for Each Net - -Each folder contains the required deployment files for it's net, work in the folder that has the name of the network you want to create snapshots for. - -What does each file do: -- `.env` - contains environment files maintaned by Threefold Tech -- `.gitignore` - has a list of files to ignore once the repo has been cloned. This has the purpose to not have uncommited changes to files when working in this repo -- `.secrets.env-examples` - is where you have to add all your unique environment variables -- `create_snapshot.sh` - script to create a snapshot (used by cron) -- `docker-compose.yml` - has all the required docker-compose configuration to deploy a working Grid stack -- `open_logs_tmux.sh` - opens all the docker logs in tmux sessions -- `typesBundle.json` - contains data for the Graphql indexer and is not to be touched -- `startall.sh` - starts all the (already deployed) containers -- `stopall.sh` - stops all the (already deployed) containers - - -## Deployment - -`cd` into the correct folder for the network your deploying for, our example uses mainnet. - -```sh -cd mainnet -cp .secrets.env-example .secrets.env -``` - -Open `.secrets.env` and add your generated subkey node-key - -Check if all environment variables are correct. -``` -docker compose --env-file .secrets.env --env-file .env config -``` - -Deploy the snapshot backend. Depending on the disk iops available, it can take up until a week to sync from block 0. - -```sh -docker compose --env-file .secrets.env --env-file .env up -d -``` - -## Script - -Add the appropriate script to cron with an interval you want the snapshots to be created - -```sh -mkdir /var/log/snapshots -touch /var/log/snapshots/snapshots-cron.log -crontab -e -``` - -```sh -0 1 * * * sh /root/code/grid_deployment/grid-snapshots/mainnet/create_snapshot.sh > /var/log/snapshots/snapshots-cron.log 2>&1 -``` - -This example will execute the script every day at 1 AM and send the logs to /var/log/snapshots/snapshots-cron.log - - -## Rsync - -We use rsync to expose the snapshots to the community. To setup a public rsync server create and edit the following file: - -- File `/etc/rsyncd.conf`: - -```sh -pid file = /var/run/rsyncd.pid -lock file = /var/run/rsync.lock -log file = /var/log/rsync.log -port = 34873 -max connections = 20 -exclude = lost+found/ -transfer logging = yes -use chroot = yes -reverse lookup = no - -[gridsnapshots] -path = /storage/rsync-public/mainnet -comment = THREEFOLD GRID MAINNET SNAPSHOTS -read only = true -timeout = 300 -list = false - -[gridsnapshotstest] -path = /storage/rsync-public/testnet -comment = THREEFOLD GRID TESTNET SNAPSHOTS -read only = true -timeout = 300 -list = false - -[gridsnapshotsdev] -path = /storage/rsync-public/devnet -comment = THREEFOLD GRID DEVNET SNAPSHOTS -read only = true -timeout = 300 -list = false -``` - -Start and enable the service with systemd: - -```sh -systemctl start rsync -systemctl enable rsync -systemctl status rsync -``` +> Fill info with validator snapshot process \ No newline at end of file From 796019612b49a5a1dac2cc5656e7fea4358e27e0 Mon Sep 17 00:00:00 2001 From: mik-tf Date: Fri, 18 Oct 2024 11:31:32 -0400 Subject: [PATCH 7/7] set grid-snapshots to grid-backend-snapshots in all repo --- README.md | 4 ++-- {grid-snapshots => grid-backend-snapshots}/devnet/.env | 0 .../devnet/.gitignore | 0 .../devnet/.secrets.env-example | 0 .../devnet/create_snapshot.sh | 8 ++++---- .../devnet/docker-compose.yml | 0 .../devnet/open_logs_tmux.sh | 0 .../devnet/setup_db.sh | 0 .../devnet/startall.sh | 0 .../devnet/stopall.sh | 0 .../devnet/typesBundle.json | 0 {grid-snapshots => grid-backend-snapshots}/mainnet/.env | 0 .../mainnet/.gitignore | 0 .../mainnet/.secrets.env-example | 0 .../mainnet/create_snapshot.sh | 0 .../mainnet/docker-compose.yml | 0 .../mainnet/open_logs_tmux.sh | 0 .../mainnet/setup_db.sh | 0 .../mainnet/startall.sh | 0 .../mainnet/stopall.sh | 0 .../mainnet/typesBundle.json | 0 {grid-snapshots => grid-backend-snapshots}/qanet/.env | 0 .../qanet/.gitignore | 0 .../qanet/.secrets.env-example | 0 .../qanet/create_snapshot.sh | 8 ++++---- .../qanet/docker-compose.yml | 0 .../qanet/open_logs_tmux.sh | 0 .../qanet/setup_db.sh | 0 .../qanet/startall.sh | 0 .../qanet/stopall.sh | 0 .../qanet/typesBundle.json | 0 {grid-snapshots => grid-backend-snapshots}/readme.md | 2 +- {grid-snapshots => grid-backend-snapshots}/testnet/.env | 0 .../testnet/.gitignore | 0 .../testnet/.secrets.env-example | 0 .../testnet/create-snapshot.sh | 8 ++++---- .../testnet/docker-compose.yml | 0 .../testnet/open_logs_tmux.sh | 0 .../testnet/setup_db.sh | 0 .../testnet/startall.sh | 0 .../testnet/stopall.sh | 0 .../testnet/typesBundle.json | 0 tfchain-validator-snapshots/devnet/create_snapshot.sh | 4 ++-- tfchain-validator-snapshots/mainnet/create_snapshot.sh | 4 ++-- tfchain-validator-snapshots/qanet/create_snapshot.sh | 4 ++-- tfchain-validator-snapshots/testnet/create_snapshot.sh | 4 ++-- 46 files changed, 23 insertions(+), 23 deletions(-) rename {grid-snapshots => grid-backend-snapshots}/devnet/.env (100%) rename {grid-snapshots => grid-backend-snapshots}/devnet/.gitignore (100%) rename {grid-snapshots => grid-backend-snapshots}/devnet/.secrets.env-example (100%) rename {grid-snapshots => grid-backend-snapshots}/devnet/create_snapshot.sh (91%) rename {grid-snapshots => grid-backend-snapshots}/devnet/docker-compose.yml (100%) rename {grid-snapshots => grid-backend-snapshots}/devnet/open_logs_tmux.sh (100%) rename {grid-snapshots => grid-backend-snapshots}/devnet/setup_db.sh (100%) rename {grid-snapshots => grid-backend-snapshots}/devnet/startall.sh (100%) rename {grid-snapshots => grid-backend-snapshots}/devnet/stopall.sh (100%) rename {grid-snapshots => grid-backend-snapshots}/devnet/typesBundle.json (100%) rename {grid-snapshots => grid-backend-snapshots}/mainnet/.env (100%) rename {grid-snapshots => grid-backend-snapshots}/mainnet/.gitignore (100%) rename {grid-snapshots => grid-backend-snapshots}/mainnet/.secrets.env-example (100%) rename {grid-snapshots => grid-backend-snapshots}/mainnet/create_snapshot.sh (100%) rename {grid-snapshots => grid-backend-snapshots}/mainnet/docker-compose.yml (100%) rename {grid-snapshots => grid-backend-snapshots}/mainnet/open_logs_tmux.sh (100%) rename {grid-snapshots => grid-backend-snapshots}/mainnet/setup_db.sh (100%) rename {grid-snapshots => grid-backend-snapshots}/mainnet/startall.sh (100%) rename {grid-snapshots => grid-backend-snapshots}/mainnet/stopall.sh (100%) rename {grid-snapshots => grid-backend-snapshots}/mainnet/typesBundle.json (100%) rename {grid-snapshots => grid-backend-snapshots}/qanet/.env (100%) rename {grid-snapshots => grid-backend-snapshots}/qanet/.gitignore (100%) rename {grid-snapshots => grid-backend-snapshots}/qanet/.secrets.env-example (100%) rename {grid-snapshots => grid-backend-snapshots}/qanet/create_snapshot.sh (91%) rename {grid-snapshots => grid-backend-snapshots}/qanet/docker-compose.yml (100%) rename {grid-snapshots => grid-backend-snapshots}/qanet/open_logs_tmux.sh (100%) rename {grid-snapshots => grid-backend-snapshots}/qanet/setup_db.sh (100%) rename {grid-snapshots => grid-backend-snapshots}/qanet/startall.sh (100%) rename {grid-snapshots => grid-backend-snapshots}/qanet/stopall.sh (100%) rename {grid-snapshots => grid-backend-snapshots}/qanet/typesBundle.json (100%) rename {grid-snapshots => grid-backend-snapshots}/readme.md (97%) rename {grid-snapshots => grid-backend-snapshots}/testnet/.env (100%) rename {grid-snapshots => grid-backend-snapshots}/testnet/.gitignore (100%) rename {grid-snapshots => grid-backend-snapshots}/testnet/.secrets.env-example (100%) rename {grid-snapshots => grid-backend-snapshots}/testnet/create-snapshot.sh (91%) rename {grid-snapshots => grid-backend-snapshots}/testnet/docker-compose.yml (100%) rename {grid-snapshots => grid-backend-snapshots}/testnet/open_logs_tmux.sh (100%) rename {grid-snapshots => grid-backend-snapshots}/testnet/setup_db.sh (100%) rename {grid-snapshots => grid-backend-snapshots}/testnet/startall.sh (100%) rename {grid-snapshots => grid-backend-snapshots}/testnet/stopall.sh (100%) rename {grid-snapshots => grid-backend-snapshots}/testnet/typesBundle.json (100%) diff --git a/README.md b/README.md index 0344cb6..fe933f5 100644 --- a/README.md +++ b/README.md @@ -53,8 +53,8 @@ Have a look at the [documentation](./tfchain-validator/readme.md) to get started Daily TFGrid and TFChain validator snapshots can be found at [https://bknd.snapshot.grid.tf/](https://bknd.snapshot.grid.tf/). -To set up your own TFGrid snapshot creation, read [this documentation](./grid-snapshots/readme.md). +To set up your own TFGrid snapshot creation, read [this documentation](./grid-backend-snapshots/readme.md). To set up your own TFChain validator snapshot creation, read [this documentation](./tfchain-validator-snapshots/readme.md). -> Note: You can also use Rsync to download the snapshots. Read [this section](./grid-snapshots/readme.md#threefold-public-rsync) for more information. \ No newline at end of file +> Note: You can also use Rsync to download the snapshots. Read [this section](./grid-backend-snapshots/readme.md#threefold-public-rsync) for more information. \ No newline at end of file diff --git a/grid-snapshots/devnet/.env b/grid-backend-snapshots/devnet/.env similarity index 100% rename from grid-snapshots/devnet/.env rename to grid-backend-snapshots/devnet/.env diff --git a/grid-snapshots/devnet/.gitignore b/grid-backend-snapshots/devnet/.gitignore similarity index 100% rename from grid-snapshots/devnet/.gitignore rename to grid-backend-snapshots/devnet/.gitignore diff --git a/grid-snapshots/devnet/.secrets.env-example b/grid-backend-snapshots/devnet/.secrets.env-example similarity index 100% rename from grid-snapshots/devnet/.secrets.env-example rename to grid-backend-snapshots/devnet/.secrets.env-example diff --git a/grid-snapshots/devnet/create_snapshot.sh b/grid-backend-snapshots/devnet/create_snapshot.sh similarity index 91% rename from grid-snapshots/devnet/create_snapshot.sh rename to grid-backend-snapshots/devnet/create_snapshot.sh index 1ecf4bb..ab2b5a4 100644 --- a/grid-snapshots/devnet/create_snapshot.sh +++ b/grid-backend-snapshots/devnet/create_snapshot.sh @@ -65,7 +65,7 @@ ln -s processor-devnet-$(date '+%Y-%m-%d').tar.gz processor-devnet-latest.tar.gz ## Send over to Grid-snapshots server and set ln -scp /storage/rsync-public/tfchain-devnet-$(date '+%Y-%m-%d').tar.gz grid-snapshots:/storage/rsync-public/devnet/ -scp /storage/rsync-public/indexer-devnet-$(date '+%Y-%m-%d').tar.gz grid-snapshots:/storage/rsync-public/devnet/ -scp /storage/rsync-public/processor-devnet-$(date '+%Y-%m-%d').tar.gz grid-snapshots:/storage/rsync-public/devnet/ -ssh grid-snapshots sh /opt/snapshots/devnet-set-ln.sh +scp /storage/rsync-public/tfchain-devnet-$(date '+%Y-%m-%d').tar.gz grid-backend-snapshots:/storage/rsync-public/devnet/ +scp /storage/rsync-public/indexer-devnet-$(date '+%Y-%m-%d').tar.gz grid-backend-snapshots:/storage/rsync-public/devnet/ +scp /storage/rsync-public/processor-devnet-$(date '+%Y-%m-%d').tar.gz grid-backend-snapshots:/storage/rsync-public/devnet/ +ssh grid-backend-snapshots sh /opt/snapshots/devnet-set-ln.sh diff --git a/grid-snapshots/devnet/docker-compose.yml b/grid-backend-snapshots/devnet/docker-compose.yml similarity index 100% rename from grid-snapshots/devnet/docker-compose.yml rename to grid-backend-snapshots/devnet/docker-compose.yml diff --git a/grid-snapshots/devnet/open_logs_tmux.sh b/grid-backend-snapshots/devnet/open_logs_tmux.sh similarity index 100% rename from grid-snapshots/devnet/open_logs_tmux.sh rename to grid-backend-snapshots/devnet/open_logs_tmux.sh diff --git a/grid-snapshots/devnet/setup_db.sh b/grid-backend-snapshots/devnet/setup_db.sh similarity index 100% rename from grid-snapshots/devnet/setup_db.sh rename to grid-backend-snapshots/devnet/setup_db.sh diff --git a/grid-snapshots/devnet/startall.sh b/grid-backend-snapshots/devnet/startall.sh similarity index 100% rename from grid-snapshots/devnet/startall.sh rename to grid-backend-snapshots/devnet/startall.sh diff --git a/grid-snapshots/devnet/stopall.sh b/grid-backend-snapshots/devnet/stopall.sh similarity index 100% rename from grid-snapshots/devnet/stopall.sh rename to grid-backend-snapshots/devnet/stopall.sh diff --git a/grid-snapshots/devnet/typesBundle.json b/grid-backend-snapshots/devnet/typesBundle.json similarity index 100% rename from grid-snapshots/devnet/typesBundle.json rename to grid-backend-snapshots/devnet/typesBundle.json diff --git a/grid-snapshots/mainnet/.env b/grid-backend-snapshots/mainnet/.env similarity index 100% rename from grid-snapshots/mainnet/.env rename to grid-backend-snapshots/mainnet/.env diff --git a/grid-snapshots/mainnet/.gitignore b/grid-backend-snapshots/mainnet/.gitignore similarity index 100% rename from grid-snapshots/mainnet/.gitignore rename to grid-backend-snapshots/mainnet/.gitignore diff --git a/grid-snapshots/mainnet/.secrets.env-example b/grid-backend-snapshots/mainnet/.secrets.env-example similarity index 100% rename from grid-snapshots/mainnet/.secrets.env-example rename to grid-backend-snapshots/mainnet/.secrets.env-example diff --git a/grid-snapshots/mainnet/create_snapshot.sh b/grid-backend-snapshots/mainnet/create_snapshot.sh similarity index 100% rename from grid-snapshots/mainnet/create_snapshot.sh rename to grid-backend-snapshots/mainnet/create_snapshot.sh diff --git a/grid-snapshots/mainnet/docker-compose.yml b/grid-backend-snapshots/mainnet/docker-compose.yml similarity index 100% rename from grid-snapshots/mainnet/docker-compose.yml rename to grid-backend-snapshots/mainnet/docker-compose.yml diff --git a/grid-snapshots/mainnet/open_logs_tmux.sh b/grid-backend-snapshots/mainnet/open_logs_tmux.sh similarity index 100% rename from grid-snapshots/mainnet/open_logs_tmux.sh rename to grid-backend-snapshots/mainnet/open_logs_tmux.sh diff --git a/grid-snapshots/mainnet/setup_db.sh b/grid-backend-snapshots/mainnet/setup_db.sh similarity index 100% rename from grid-snapshots/mainnet/setup_db.sh rename to grid-backend-snapshots/mainnet/setup_db.sh diff --git a/grid-snapshots/mainnet/startall.sh b/grid-backend-snapshots/mainnet/startall.sh similarity index 100% rename from grid-snapshots/mainnet/startall.sh rename to grid-backend-snapshots/mainnet/startall.sh diff --git a/grid-snapshots/mainnet/stopall.sh b/grid-backend-snapshots/mainnet/stopall.sh similarity index 100% rename from grid-snapshots/mainnet/stopall.sh rename to grid-backend-snapshots/mainnet/stopall.sh diff --git a/grid-snapshots/mainnet/typesBundle.json b/grid-backend-snapshots/mainnet/typesBundle.json similarity index 100% rename from grid-snapshots/mainnet/typesBundle.json rename to grid-backend-snapshots/mainnet/typesBundle.json diff --git a/grid-snapshots/qanet/.env b/grid-backend-snapshots/qanet/.env similarity index 100% rename from grid-snapshots/qanet/.env rename to grid-backend-snapshots/qanet/.env diff --git a/grid-snapshots/qanet/.gitignore b/grid-backend-snapshots/qanet/.gitignore similarity index 100% rename from grid-snapshots/qanet/.gitignore rename to grid-backend-snapshots/qanet/.gitignore diff --git a/grid-snapshots/qanet/.secrets.env-example b/grid-backend-snapshots/qanet/.secrets.env-example similarity index 100% rename from grid-snapshots/qanet/.secrets.env-example rename to grid-backend-snapshots/qanet/.secrets.env-example diff --git a/grid-snapshots/qanet/create_snapshot.sh b/grid-backend-snapshots/qanet/create_snapshot.sh similarity index 91% rename from grid-snapshots/qanet/create_snapshot.sh rename to grid-backend-snapshots/qanet/create_snapshot.sh index 2b6f2b3..96f3f82 100644 --- a/grid-snapshots/qanet/create_snapshot.sh +++ b/grid-backend-snapshots/qanet/create_snapshot.sh @@ -65,7 +65,7 @@ ln -s processor-qanet-$(date '+%Y-%m-%d').tar.gz processor-qanet-latest.tar.gz ## Send over to Grid-snapshots server and set ln -scp /storage/rsync-public/tfchain-qanet-$(date '+%Y-%m-%d').tar.gz grid-snapshots:/storage/rsync-public/qanet/ -scp /storage/rsync-public/indexer-qanet-$(date '+%Y-%m-%d').tar.gz grid-snapshots:/storage/rsync-public/qanet/ -scp /storage/rsync-public/processor-qanet-$(date '+%Y-%m-%d').tar.gz grid-snapshots:/storage/rsync-public/qanet/ -ssh grid-snapshots sh /opt/snapshots/qanet-set-ln.sh +scp /storage/rsync-public/tfchain-qanet-$(date '+%Y-%m-%d').tar.gz grid-backend-snapshots:/storage/rsync-public/qanet/ +scp /storage/rsync-public/indexer-qanet-$(date '+%Y-%m-%d').tar.gz grid-backend-snapshots:/storage/rsync-public/qanet/ +scp /storage/rsync-public/processor-qanet-$(date '+%Y-%m-%d').tar.gz grid-backend-snapshots:/storage/rsync-public/qanet/ +ssh grid-backend-snapshots sh /opt/snapshots/qanet-set-ln.sh diff --git a/grid-snapshots/qanet/docker-compose.yml b/grid-backend-snapshots/qanet/docker-compose.yml similarity index 100% rename from grid-snapshots/qanet/docker-compose.yml rename to grid-backend-snapshots/qanet/docker-compose.yml diff --git a/grid-snapshots/qanet/open_logs_tmux.sh b/grid-backend-snapshots/qanet/open_logs_tmux.sh similarity index 100% rename from grid-snapshots/qanet/open_logs_tmux.sh rename to grid-backend-snapshots/qanet/open_logs_tmux.sh diff --git a/grid-snapshots/qanet/setup_db.sh b/grid-backend-snapshots/qanet/setup_db.sh similarity index 100% rename from grid-snapshots/qanet/setup_db.sh rename to grid-backend-snapshots/qanet/setup_db.sh diff --git a/grid-snapshots/qanet/startall.sh b/grid-backend-snapshots/qanet/startall.sh similarity index 100% rename from grid-snapshots/qanet/startall.sh rename to grid-backend-snapshots/qanet/startall.sh diff --git a/grid-snapshots/qanet/stopall.sh b/grid-backend-snapshots/qanet/stopall.sh similarity index 100% rename from grid-snapshots/qanet/stopall.sh rename to grid-backend-snapshots/qanet/stopall.sh diff --git a/grid-snapshots/qanet/typesBundle.json b/grid-backend-snapshots/qanet/typesBundle.json similarity index 100% rename from grid-snapshots/qanet/typesBundle.json rename to grid-backend-snapshots/qanet/typesBundle.json diff --git a/grid-snapshots/readme.md b/grid-backend-snapshots/readme.md similarity index 97% rename from grid-snapshots/readme.md rename to grid-backend-snapshots/readme.md index f785326..1472f40 100644 --- a/grid-snapshots/readme.md +++ b/grid-backend-snapshots/readme.md @@ -121,7 +121,7 @@ crontab -e ``` ```sh -0 1 * * * sh /root/code/grid_deployment/grid-snapshots/mainnet/create_snapshot.sh > /var/log/snapshots/snapshots-cron.log 2>&1 +0 1 * * * sh /root/code/grid_deployment/grid-backend-snapshots/mainnet/create_snapshot.sh > /var/log/snapshots/snapshots-cron.log 2>&1 ``` This example will execute the script every day at 1 AM and send the logs to /var/log/snapshots/snapshots-cron.log diff --git a/grid-snapshots/testnet/.env b/grid-backend-snapshots/testnet/.env similarity index 100% rename from grid-snapshots/testnet/.env rename to grid-backend-snapshots/testnet/.env diff --git a/grid-snapshots/testnet/.gitignore b/grid-backend-snapshots/testnet/.gitignore similarity index 100% rename from grid-snapshots/testnet/.gitignore rename to grid-backend-snapshots/testnet/.gitignore diff --git a/grid-snapshots/testnet/.secrets.env-example b/grid-backend-snapshots/testnet/.secrets.env-example similarity index 100% rename from grid-snapshots/testnet/.secrets.env-example rename to grid-backend-snapshots/testnet/.secrets.env-example diff --git a/grid-snapshots/testnet/create-snapshot.sh b/grid-backend-snapshots/testnet/create-snapshot.sh similarity index 91% rename from grid-snapshots/testnet/create-snapshot.sh rename to grid-backend-snapshots/testnet/create-snapshot.sh index 1a3af0f..00abf32 100644 --- a/grid-snapshots/testnet/create-snapshot.sh +++ b/grid-backend-snapshots/testnet/create-snapshot.sh @@ -65,7 +65,7 @@ ln -s processor-testnet-$(date '+%Y-%m-%d').tar.gz processor-testnet-latest.tar. ## Send over to Grid-snapshots server and set ln -scp /storage/rsync-public/tfchain-testnet-$(date '+%Y-%m-%d').tar.gz grid-snapshots:/storage/rsync-public/testnet/ -scp /storage/rsync-public/indexer-testnet-$(date '+%Y-%m-%d').tar.gz grid-snapshots:/storage/rsync-public/testnet/ -scp /storage/rsync-public/processor-testnet-$(date '+%Y-%m-%d').tar.gz grid-snapshots:/storage/rsync-public/testnet/ -ssh grid-snapshots sh /opt/snapshots/testnet-set-ln.sh +scp /storage/rsync-public/tfchain-testnet-$(date '+%Y-%m-%d').tar.gz grid-backend-snapshots:/storage/rsync-public/testnet/ +scp /storage/rsync-public/indexer-testnet-$(date '+%Y-%m-%d').tar.gz grid-backend-snapshots:/storage/rsync-public/testnet/ +scp /storage/rsync-public/processor-testnet-$(date '+%Y-%m-%d').tar.gz grid-backend-snapshots:/storage/rsync-public/testnet/ +ssh grid-backend-snapshots sh /opt/snapshots/testnet-set-ln.sh diff --git a/grid-snapshots/testnet/docker-compose.yml b/grid-backend-snapshots/testnet/docker-compose.yml similarity index 100% rename from grid-snapshots/testnet/docker-compose.yml rename to grid-backend-snapshots/testnet/docker-compose.yml diff --git a/grid-snapshots/testnet/open_logs_tmux.sh b/grid-backend-snapshots/testnet/open_logs_tmux.sh similarity index 100% rename from grid-snapshots/testnet/open_logs_tmux.sh rename to grid-backend-snapshots/testnet/open_logs_tmux.sh diff --git a/grid-snapshots/testnet/setup_db.sh b/grid-backend-snapshots/testnet/setup_db.sh similarity index 100% rename from grid-snapshots/testnet/setup_db.sh rename to grid-backend-snapshots/testnet/setup_db.sh diff --git a/grid-snapshots/testnet/startall.sh b/grid-backend-snapshots/testnet/startall.sh similarity index 100% rename from grid-snapshots/testnet/startall.sh rename to grid-backend-snapshots/testnet/startall.sh diff --git a/grid-snapshots/testnet/stopall.sh b/grid-backend-snapshots/testnet/stopall.sh similarity index 100% rename from grid-snapshots/testnet/stopall.sh rename to grid-backend-snapshots/testnet/stopall.sh diff --git a/grid-snapshots/testnet/typesBundle.json b/grid-backend-snapshots/testnet/typesBundle.json similarity index 100% rename from grid-snapshots/testnet/typesBundle.json rename to grid-backend-snapshots/testnet/typesBundle.json diff --git a/tfchain-validator-snapshots/devnet/create_snapshot.sh b/tfchain-validator-snapshots/devnet/create_snapshot.sh index 51fa3b4..4b32b68 100644 --- a/tfchain-validator-snapshots/devnet/create_snapshot.sh +++ b/tfchain-validator-snapshots/devnet/create_snapshot.sh @@ -21,5 +21,5 @@ rm tfchain-devnet-validator-latest.tar.gz ln -s tfchain-devnet-validator-$(date '+%Y-%m-%d').tar.gz tfchain-devnet-validator-latest.tar.gz ## Send over to Grid-snapshots server and set ln -scp /storage/rsync-public/tfchain-devnet-validator-$(date '+%Y-%m-%d').tar.gz grid-snapshots:/storage/rsync-public/devnet/ -ssh grid-snapshots sh /opt/snapshots/devnet-validator-set-ln.sh +scp /storage/rsync-public/tfchain-devnet-validator-$(date '+%Y-%m-%d').tar.gz grid-backend-snapshots:/storage/rsync-public/devnet/ +ssh grid-backend-snapshots sh /opt/snapshots/devnet-validator-set-ln.sh diff --git a/tfchain-validator-snapshots/mainnet/create_snapshot.sh b/tfchain-validator-snapshots/mainnet/create_snapshot.sh index 0e61b3e..b9fe7ae 100644 --- a/tfchain-validator-snapshots/mainnet/create_snapshot.sh +++ b/tfchain-validator-snapshots/mainnet/create_snapshot.sh @@ -21,5 +21,5 @@ rm tfchain-mainnet-validator-latest.tar.gz ln -s tfchain-mainnet-validator-$(date '+%Y-%m-%d').tar.gz tfchain-mainnet-validator-latest.tar.gz ## Send over to Grid-snapshots server and set ln -scp /storage/rsync-public/tfchain-mainnet-validator-$(date '+%Y-%m-%d').tar.gz grid-snapshots:/storage/rsync-public/mainnet/ -ssh grid-snapshots sh /opt/snapshots/mainnet-validator-set-ln.sh +scp /storage/rsync-public/tfchain-mainnet-validator-$(date '+%Y-%m-%d').tar.gz grid-backend-snapshots:/storage/rsync-public/mainnet/ +ssh grid-backend-snapshots sh /opt/snapshots/mainnet-validator-set-ln.sh diff --git a/tfchain-validator-snapshots/qanet/create_snapshot.sh b/tfchain-validator-snapshots/qanet/create_snapshot.sh index 469ffa7..bb32e79 100644 --- a/tfchain-validator-snapshots/qanet/create_snapshot.sh +++ b/tfchain-validator-snapshots/qanet/create_snapshot.sh @@ -21,5 +21,5 @@ rm tfchain-qanet-validator-latest.tar.gz ln -s tfchain-qanet-validator-$(date '+%Y-%m-%d').tar.gz tfchain-qanet-validator-latest.tar.gz ## Send over to Grid-snapshots server and set ln -scp /storage/rsync-public/tfchain-qanet-validator-$(date '+%Y-%m-%d').tar.gz grid-snapshots:/storage/rsync-public/qanet/ -ssh grid-snapshots sh /opt/snapshots/qanet-validator-set-ln.sh +scp /storage/rsync-public/tfchain-qanet-validator-$(date '+%Y-%m-%d').tar.gz grid-backend-snapshots:/storage/rsync-public/qanet/ +ssh grid-backend-snapshots sh /opt/snapshots/qanet-validator-set-ln.sh diff --git a/tfchain-validator-snapshots/testnet/create_snapshot.sh b/tfchain-validator-snapshots/testnet/create_snapshot.sh index b588046..6d05a13 100644 --- a/tfchain-validator-snapshots/testnet/create_snapshot.sh +++ b/tfchain-validator-snapshots/testnet/create_snapshot.sh @@ -21,5 +21,5 @@ rm tfchain-testnet-validator-latest.tar.gz ln -s tfchain-testnet-validator-$(date '+%Y-%m-%d').tar.gz tfchain-testnet-validator-latest.tar.gz ## Send over to Grid-snapshots server and set ln -scp /storage/rsync-public/tfchain-testnet-validator-$(date '+%Y-%m-%d').tar.gz grid-snapshots:/storage/rsync-public/testnet/ -ssh grid-snapshots sh /opt/snapshots/testnet-validator-set-ln.sh +scp /storage/rsync-public/tfchain-testnet-validator-$(date '+%Y-%m-%d').tar.gz grid-backend-snapshots:/storage/rsync-public/testnet/ +ssh grid-backend-snapshots sh /opt/snapshots/testnet-validator-set-ln.sh