diff --git a/community-edition/.gitignore b/community-edition/.gitignore index 2547855b..55b5dff7 100644 --- a/community-edition/.gitignore +++ b/community-edition/.gitignore @@ -1,4 +1,5 @@ /node_modules /hcce.yaml /ssl_script/cbb.yaml -.DS_Store \ No newline at end of file +/data_backups +.DS_Store diff --git a/community-edition/backup_script/index.js b/community-edition/backup_script/index.js new file mode 100644 index 00000000..e9bd1cbe --- /dev/null +++ b/community-edition/backup_script/index.js @@ -0,0 +1,50 @@ +const execSync = require('child_process').execSync; +const fs = require("fs"); +const path = require("path"); +const YAML = require("yaml"); +const utils = require("../utils"); + +// read config +const config = utils.readConfig(); +const processedConfig = YAML.parse( + utils.replacePlaceholders(YAML.stringify(config), config), + {"schema": "yaml-1.1"} // required to load yes/no as boolean values +); + +// get backup paths +const rootDataBackupPath = path.join(process.cwd(), "data_backups"); +const dataBackupPath = path.join(rootDataBackupPath, `data_backup_${Date.now()}`); +const reticulumStoragePath = path.join(dataBackupPath, "reticulum_storage_data"); +const pgDumpSQLPath = path.join(dataBackupPath, "pg_dump.sql"); + +// get pod names +let reticulumPodName = execSync(`kubectl get pods -l=app=reticulum -n ${processedConfig.Namespace} --output jsonpath='{.items[0].metadata.name}'`); +let pgsqlPodName = execSync(`kubectl get pods -l=app=pgsql -n ${processedConfig.Namespace} --output jsonpath='{.items[*].metadata.name}'`); +// strip out the single quotes that Windows adds in +reticulumPodName = reticulumPodName.toString().replaceAll("'", ""); +pgsqlPodName = pgsqlPodName.toString().replaceAll("'", ""); + + +// make backups folder (if needed) +if (!fs.existsSync(rootDataBackupPath)) { + fs.mkdirSync(rootDataBackupPath); + } + +// download reticulum storage +// note: relative paths must be used for kubectl cp on windows due to this bug: https://github.com/kubernetes/kubernetes/issues/101985 +const reticulumOutputPath = path.relative(process.cwd(), reticulumStoragePath); +console.log(`copying reticulum files to ${reticulumOutputPath}`); +execSync(`kubectl cp --retries=-1 ${reticulumPodName}:/storage ${reticulumOutputPath} -n ${processedConfig.Namespace}`, { env: { ...process.env, KUBECTL_REMOTE_COMMAND_WEBSOCKETS: false } }); + +if (pgsqlPodName) { + // create and download dump of pgsql database + // note: relative paths must be used for kubectl cp on windows due to this bug: https://github.com/kubernetes/kubernetes/issues/101985 + console.log(`dumping pgsql`); + execSync(`kubectl exec ${pgsqlPodName} -n ${processedConfig.Namespace} -- /bin/pg_dump -c ${processedConfig.PGRST_DB_URI} -f /root/pg_dump.sql`); + const pgsqlOutputPath = path.relative(process.cwd(), pgDumpSQLPath); + console.log(`copying dump to ${pgsqlOutputPath}`); + execSync(`kubectl cp --retries=-1 ${pgsqlPodName}:/root/pg_dump.sql ${pgsqlOutputPath} -n ${processedConfig.Namespace}`, { env: { ...process.env, KUBECTL_REMOTE_COMMAND_WEBSOCKETS: false } }); + execSync(`kubectl exec ${pgsqlPodName} -n ${processedConfig.Namespace} -- /bin/rm /root/pg_dump.sql`); +} else { + console.warn('not backing up pgsql; pod not found'); +} diff --git a/community-edition/get_ip/index.js b/community-edition/get_ip/index.js index 70f4d92d..145b5643 100644 --- a/community-edition/get_ip/index.js +++ b/community-edition/get_ip/index.js @@ -4,13 +4,22 @@ const { spawnSync } = require("node:child_process"); const config = utils.readConfig(); const { stdout } = spawnSync( "kubectl", - ["-n", config.Namespace, "get", "svc", "lb", "-o", "json"], + ["get", "svc", "--field-selector", "spec.type=LoadBalancer", "-A", "-o", "json"], { stdio: ["pipe", "pipe", "inherit"] } ); const output = JSON.parse(stdout); -const ipAddr = output.status.loadBalancer?.ingress?.[0]?.ip; -if (ipAddr) { - console.log("load balancer external IP address:", ipAddr); -} else { - console.log("load balancer not running yet:", output.status.loadBalancer); +if (output.items.length === 0) { + console.warn("can't determine external IP address: no load balancers in cluster"); + process.exit(1); +} +for (const item of output.items) { + const name = item.metadata.name; + const namespace = item.metadata.namespace; + const status = item.status; + const addr = status?.loadBalancer?.ingress?.[0]?.ip || status?.loadBalancer?.ingress?.[0]?.hostname; + if (addr) { + console.log(`load balancer “${name}” in namespace “${namespace}” external address: ${addr}`); + } else { + console.log(`load balancer “${name}” in namespace “${namespace}” not running yet:`, status); + } } diff --git a/community-edition/package-lock.json b/community-edition/package-lock.json index cba91351..a90ca274 100644 --- a/community-edition/package-lock.json +++ b/community-edition/package-lock.json @@ -10,6 +10,7 @@ "license": "ISC", "dependencies": { "inquirer": "^10.0.1", + "junk": "^4.0.1", "node-forge": "^1.3.1", "openssl-nodejs": "^1.0.5", "pem-jwk": "^2.0.0", @@ -367,6 +368,17 @@ "node": ">=8" } }, + "node_modules/junk": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/junk/-/junk-4.0.1.tgz", + "integrity": "sha512-Qush0uP+G8ZScpGMZvHUiRfI0YBWuB3gVBYlI0v0vvOJt5FLicco+IkP0a50LqTTQhmts/m6tP5SWE+USyIvcQ==", + "engines": { + "node": ">=12.20" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/minimalistic-assert": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/minimalistic-assert/-/minimalistic-assert-1.0.1.tgz", diff --git a/community-edition/package.json b/community-edition/package.json index 605729bc..287792ed 100644 --- a/community-edition/package.json +++ b/community-edition/package.json @@ -8,6 +8,8 @@ "apply": "node apply/index.js && node get_ip/index.js", "get-ip": "node get_ip/index.js", "gen-ssl": "node ssl_script/index.js", + "backup": "node backup_script/index.js", + "restore-backup": "node restore_backup_script/index.js", "test": "echo \"Error: no test specified\" && exit 1" }, "keywords": [], @@ -16,6 +18,7 @@ "description": "", "dependencies": { "inquirer": "^10.0.1", + "junk": "^4.0.1", "node-forge": "^1.3.1", "openssl-nodejs": "^1.0.5", "pem-jwk": "^2.0.0", diff --git a/community-edition/readme.md b/community-edition/readme.md index 765dea8d..43a5750b 100644 --- a/community-edition/readme.md +++ b/community-edition/readme.md @@ -131,6 +131,15 @@ If you just need to get the external IP address of your load balancer, run `npm run get-ip` +### Backing up and restoring your instance + +Use `npm run backup` to back up your instance. The backup will be timestamped and placed in a `data_backups` folder. + +Use `npm run restore-backup data_backup_1234567890123` to restore a backup to your instance. If you don't specify a backup, and just use `npm run restore-backup`, it will default to the latest backup. +The `hcce.yaml` file in the `community-edition` directory must match your instance. + +If you run an external database instead of the `pgsql` pod, the scripts will only back up and restore the reticulum files. + ## Guides from the Hubs Team and Community ### 1. Beginner's Guide to CE diff --git a/community-edition/restore_backup_script/index.js b/community-edition/restore_backup_script/index.js new file mode 100644 index 00000000..25fc55f8 --- /dev/null +++ b/community-edition/restore_backup_script/index.js @@ -0,0 +1,139 @@ +(async () => { + const execSync = require('child_process').execSync; + const fs = require("fs"); + const path = require("path"); + const YAML = require("yaml"); + const utils = require("../utils.js"); + const junk = await import("junk"); + + // get command line arguments + const args = process.argv.slice(2); + + // read config + const config = utils.readConfig(); + const processedConfig = YAML.parse( + utils.replacePlaceholders(YAML.stringify(config), config), + {"schema": "yaml-1.1"} // required to load yes/no as boolean values + ); + + // apply maintenance mode + const maintenanceModeHcceFileName = "maintenance-mode-hcce.yaml" + const hcce = utils.readTemplate("", "hcce.yaml"); + const hcceYamlDocuments = YAML.parseAllDocuments(hcce); + hcceYamlDocuments.forEach((doc, index) => { + const jsDoc = doc.toJS(); + if (jsDoc?.kind === "Ingress") { + if (!jsDoc.metadata.annotations) { + jsDoc.metadata["annotations"] = {}; + } + jsDoc.metadata.annotations["haproxy.org/request-redirect"] = `hubs-maintenance-mode.${processedConfig.HUB_DOMAIN}` + hcceYamlDocuments[index] = new YAML.Document(jsDoc); + } + }); + const maintenanceModeHcce = `${hcceYamlDocuments.map(doc => YAML.stringify(doc, {"lineWidth": 0, "directives": false})).join('---\n')}` + utils.writeOutputFile(maintenanceModeHcce, "", maintenanceModeHcceFileName); + + console.log("applying maintenance mode"); + console.log(""); + execSync(`kubectl delete deployment --all -n ${processedConfig.Namespace}`, {stdio: 'inherit'}); + execSync(`kubectl delete pods --all -n ${processedConfig.Namespace}`, {stdio: 'inherit'}); + execSync(`kubectl apply -f ${maintenanceModeHcceFileName}`, {stdio: 'inherit'}); + let pendingDeploymentNames = [] + while (true) { + let deployments = JSON.parse(execSync(`kubectl get deployment -n ${config.Namespace} -o json`)).items; + let pendingDeployments = deployments.filter(deployment => (deployment.status.readyReplicas ?? 0) < deployment.status.replicas); + + if (pendingDeployments.length) { + currentPendingDeploymentNames = pendingDeployments.map(deployment => deployment.metadata.name) + if (currentPendingDeploymentNames.toString() !== pendingDeploymentNames.toString()) { + console.log(`waiting on ${currentPendingDeploymentNames.join(", ")}`); + pendingDeploymentNames = currentPendingDeploymentNames; + } + // Wait for 1 second + Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, 1000); + } else { + console.log("maintenance mode applied") + break; + } + } + + // get backup paths + const rootDataBackupPath = path.join(process.cwd(), "data_backups"); + const backup_name = args[0] ? args[0] : + fs.readdirSync(rootDataBackupPath) + .filter(name => name.includes("data_backup")) + .sort().at(-1); + const dataBackupPath = path.join(rootDataBackupPath, backup_name); + const reticulumStoragePath = path.join(dataBackupPath, "reticulum_storage_data"); + const reticulumStorageRelativePath = path.relative(process.cwd(), reticulumStoragePath); + const pgDumpSQLPath = path.join(dataBackupPath, "pg_dump.sql"); + if (!fs.existsSync(dataBackupPath)) { + console.error("the specified backup doesn't exist"); + process.exit(1); + } + + // get pod names + let reticulumPodName = execSync(`kubectl get pods -l=app=reticulum -n ${processedConfig.Namespace} --output jsonpath='{.items[0].metadata.name}'`); + let pgsqlPodName = execSync(`kubectl get pods -l=app=pgsql -n ${processedConfig.Namespace} --output jsonpath='{.items[*].metadata.name}'`); + // strip out the single quotes that Windows adds in + reticulumPodName = reticulumPodName.toString().replaceAll("'", ""); + pgsqlPodName = pgsqlPodName.toString().replaceAll("'", ""); + + if (!pgsqlPodName) { + console.warn("pgsql pod not found"); + } + + console.log(""); + console.log("restoring backup"); + console.log(""); + + // remove any OS helper files from the reticulum storage + function remove_os_helper_files_recursive(base_path) { + if (fs.statSync(base_path).isDirectory()) { + let fs_object_names = fs.readdirSync(base_path); + + fs_object_names.forEach(fs_object_name => { + let fs_object_path = path.join(base_path, fs_object_name); + + if (junk.isJunk(fs_object_name)) { + // delete unneeded OS helper files that may have been added to the backup by the user's OS. + fs.rmSync(fs_object_path, { recursive: true, force: true }); + } else { + remove_os_helper_files_recursive(fs_object_path); + } + }); + } + } + remove_os_helper_files_recursive(reticulumStoragePath); + + // remove the reticulum pod's storage on the kubernetes cluster + execSync(`kubectl exec ${reticulumPodName} -c reticulum -n ${processedConfig.Namespace} -- /usr/bin/find /storage -mindepth 1 -delete`); + + // upload reticulum storage + // note: relative paths must be used for kubectl cp on windows due to this bug: https://github.com/kubernetes/kubernetes/issues/101985 + let fs_object_names = fs.readdirSync(reticulumStoragePath); + fs_object_names.forEach(fs_object_name => { + console.log(`restoring Reticulum '${fs_object_name}' folder`); + execSync(`kubectl cp --retries=-1 ${path.join(reticulumStorageRelativePath, fs_object_name)} ${reticulumPodName}:/storage -c reticulum -n ${processedConfig.Namespace}`, { env: { ...process.env, KUBECTL_REMOTE_COMMAND_WEBSOCKETS: false } }); + }); + + if (pgsqlPodName) { + // upload and apply the dump of the pgsql database + // note: relative paths must be used for kubectl cp on windows due to this bug: https://github.com/kubernetes/kubernetes/issues/101985 + const pgsqlInputPath = path.relative(process.cwd(), pgDumpSQLPath); + console.log(`restoring database from ${pgsqlInputPath}`); + execSync(`kubectl cp --retries=-1 ${pgsqlInputPath} ${pgsqlPodName}:/root/pg_dump.sql -n ${processedConfig.Namespace}`, { env: { ...process.env, KUBECTL_REMOTE_COMMAND_WEBSOCKETS: false } }); + execSync(`kubectl exec ${pgsqlPodName} -n ${processedConfig.Namespace} -- /bin/psql ${processedConfig.PGRST_DB_URI} -f /root/pg_dump.sql`); + execSync(`kubectl exec ${pgsqlPodName} -n ${processedConfig.Namespace} -- /bin/rm /root/pg_dump.sql`); + } else { + console.warn('not restoring database'); + } + + // restart the Hubs instance so it doesn't error out when visited and maintenance mode is no longer applied + fs.rmSync(path.join(process.cwd(), maintenanceModeHcceFileName)); + console.log(""); + console.log("restarting instance"); + execSync(`kubectl delete deployment --all -n ${processedConfig.Namespace}`, {stdio: 'inherit'}); + execSync(`kubectl delete pods --all -n ${processedConfig.Namespace}`, {stdio: 'inherit'}); + execSync(`npm run apply`, {stdio: 'inherit'}); +})();