diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..3c3629e --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +node_modules diff --git a/Dockerfile b/Dockerfile index 8148729..9ca667c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,17 +2,17 @@ FROM node:20-bookworm LABEL maintainer="team@semantic.works" -RUN apt-get update && apt-get -y upgrade && apt-get -y install git openssh-client rsync +RUN apt-get update && apt-get -y upgrade && apt-get -y install git openssh-client rsync jq -ENV MU_SPARQL_ENDPOINT 'http://database:8890/sparql' -ENV MU_APPLICATION_GRAPH 'http://mu.semte.ch/application' -ENV NODE_ENV 'production' +ENV MU_SPARQL_ENDPOINT='http://database:8890/sparql' +ENV MU_APPLICATION_GRAPH='http://mu.semte.ch/application' +ENV NODE_ENV='production' -ENV HOST '0.0.0.0' -ENV PORT '80' +ENV HOST='0.0.0.0' +ENV PORT='80' -ENV LOG_SPARQL_ALL 'true' -ENV DEBUG_AUTH_HEADERS 'true' +ENV LOG_SPARQL_ALL='true' +ENV DEBUG_AUTH_HEADERS='true' WORKDIR /usr/src/app COPY package.json /usr/src/app/package.json @@ -20,11 +20,12 @@ COPY ./scripts /app/scripts RUN npm install COPY . /usr/src/app RUN chmod +x /usr/src/app/run-development.sh +RUN chmod +x /usr/src/app/run-production.sh RUN chmod +x /usr/src/app/build-production.sh EXPOSE ${PORT} -CMD bash boot.sh +CMD ["bash", "boot.sh"] # This stuff only runs when building an image from the template ONBUILD RUN rm -Rf /app/scripts diff --git a/README.md b/README.md index 7791f75..4b703bb 100644 --- a/README.md +++ b/README.md @@ -202,8 +202,8 @@ mu.app.get('/', function( req, res ) { ``` The following helper functions are provided by the template - - `query(query) => Promise`: Function for sending queries to the triplestore - - `update(query) => Promise`: Function for sending updates to the triplestore + - `query(queryString, options) => Promise`: Function for sending queries to the triplestore. Options is an object which may include `sudo` and `scope` keys. + - `update(query, options) => Promise`: Function for sending updates to the triplestore. Options is an object which may include `sudo` and `scope` keys. - `uuid() => string`: Generates a random UUID (e.g. to construct new resource URIs) The following SPARQL escape helpers are provided to construct safe SPARQL query strings @@ -217,6 +217,49 @@ The following SPARQL escape helpers are provided to construct safe SPARQL query - `sparqlEscapeBool(value) => string`: The given value is evaluated to a boolean value in javascript. E.g. the string value `'0'` evaluates to `false` in javascript. - `sparqlEscape(value, type) => string`: Function to escape a value in SPARQL according to the given type. Type must be one of `'string'`, `'uri'`, `'int'`, `'float'`, `'date'`, `'dateTime'`, `'bool'`. +### Executing queries + +#### Example +```js +import { query } from 'mu'; + +const queryString = ` + PREFIX dbo: + SELECT ?name WHERE { + ?person a dbo:Person ; + dbo:birthPlace ; + dbo:name ?name . + } LIMIT 10 +`; + +const options = { + scope: "my-scope", + extraHeaders: { + "X-Custom-Header": "value" + }, +}; + +response = await query(queryString, options); +``` + +#### Parameters +The `query` and `update` helpers accept the same parameters. + +| Parameter | Type | Default | Description | +|--------------------------|-----------|----------------------|------------------------------------------------------------------------------------| +| `queryString` | `string` | _Required_ | The SPARQL query to execute. | +| `options` | `object` | `{}` | Optional configuration settings. | +| `options.sudo` | `boolean` | `false` | Whether to enable sudo mode (requires environment permission). | +| `options.scope` | `string` | _(Env default)_ | The authentication scope to use. Defaults to `DEFAULT_MU_AUTH_SCOPE` if available. | + +#### Experimental Parameters +| `options.sparqlEndpoint` | `string` | `MU_SPARQL_ENDPOINT` | The SPARQL endpoint to send the request to. | +| `options.extraHeaders` | `object` | `{}` | Additional headers to send in the request. | +| `options.authUser` | `string` | `undefined` | The username for HTTP authentication (optional). | +| `options.authPassword` | `string` | `undefined` | The password for HTTP authentication (optional). | +| `options.authType` | `string` | "digest"` | The type of authentication to use. (`"digest"` or `"basic"`) | + + ### Error handling The template offers [an error handler](https://expressjs.com/en/guide/error-handling.html) to send error responses in a JSON:API compliant way. The handler can be imported from `'mu'` and need to be loaded at the end. @@ -227,7 +270,7 @@ app.get('/hello', function( req, res, next ) { try { ... } catch (e) { - next(new Error('Oops, something went wrong.)) + next(new Error('Oops, something went wrong.')) } }); @@ -254,6 +297,8 @@ The following environment variables can be configured: - `MAX_BODY_SIZE` (default: `100kb`): max size of the request body. See [ExpressJS documentation](https://expressjs.com/en/resources/middleware/body-parser.html#limit). - `HOST` (default: `0.0.0.0`): The hostname you want the service to bind to. - `PORT` (default: `80`): The port you want the service to bind to. + - `ALLOW_MU_AUTH_SUDO`: Allow sudo queries when the service requests it. + - `DEFAULT_MU_AUTH_SCOPE`: Default mu-auth-scope to use for calls. #### Mounting `/config` diff --git a/babel.config.json b/babel.config.json index b8eae3d..e0192f2 100644 --- a/babel.config.json +++ b/babel.config.json @@ -4,16 +4,18 @@ { "targets": { "node": 18 - } + }, + "modules": false } ], ["@babel/preset-typescript"] ], "plugins": [ - ["@babel/plugin-proposal-decorators", { "version": "2023-05" }] + ["@babel/plugin-proposal-decorators", { "version": "2023-05" }], + ["babel-plugin-add-import-extension"] ], "ignore": [ "./node_modules", - "/usr/src/processing/build/node_modules" + "/usr/src/dist/node_modules" ] } diff --git a/boot.sh b/boot.sh index c39f0b9..f1d81b0 100755 --- a/boot.sh +++ b/boot.sh @@ -9,48 +9,5 @@ then --exec /usr/src/app/run-development.sh elif [ "$NODE_ENV" == "production" ] then - diff -rq /app /app.original > /dev/null - APP_FILES_CHANGED="$?" - diff -rq /config /config.original > /dev/null - CONFIG_FILES_CHANGED="$?" - - if [ ! -f /usr/src/build/app.js ] - then - echo "No built sources found. If you mount new sources, please set the NODE_ENV=\"development\" environment variable." - sleep 5; - exit 1; - elif [ $APP_FILES_CHANGED != "0" ] - then - echo "Built sources are not the same as sources available in /app. If you mount new sources, please set the NODE_ENV=\"development\" environment variable." - sleep 5; - exit 1; - elif [ $CONFIG_FILES_CHANGED != "0" ] - then - echo "Rebuilding sources to include /config." - - # move new configuration into app for transpilation - if [[ "$(ls -A /config 2> /dev/null)" ]] - then - cp -Rf /config/* /usr/src/app/app/config/ - fi - - # make a backup of the used configuration so we can detect changes - rm -Rf /config.original - mkdir /config.original - if [[ "$(ls -A /config 2> /dev/null)" ]] - then - cp -Rf /config/* /config.original - fi - - # transpile sources - cd /usr/src/app/ - ./transpile-sources.sh - - # boot transpiled sources - cd /usr/src/build/ - exec node ./app.js - else - cd /usr/src/build/ - exec node ./app.js - fi + /usr/src/app/run-production.sh fi diff --git a/build-production.sh b/build-production.sh index 33517d7..714f096 100755 --- a/build-production.sh +++ b/build-production.sh @@ -22,12 +22,7 @@ fi cp -r /app /app.original # Install custom packages if need be -if [ -f ./app/package.json ] -then - echo "Running npm install" - cd /usr/src/app/app/ - npm install - cd /usr/src/app/ -fi +./prepare-package-json.sh +./npm-install-dependencies.sh production ./transpile-sources.sh diff --git a/helpers.sh b/helpers.sh index f5d67e1..b5c2e7b 100644 --- a/helpers.sh +++ b/helpers.sh @@ -15,5 +15,5 @@ function docker-rsync() { # --numeric-ids: use uuid by number instead of by name # --info: silent output # --no-compress: no compression algorithm - rsync -aHAWXS --numeric-ids --info= --no-compress $@ + rsync -aHAWXS --numeric-ids --info= --no-compress "$@" } diff --git a/helpers/mu/index.js b/helpers/mu/index.js index 54b0c0d..d8a40f4 100644 --- a/helpers/mu/index.js +++ b/helpers/mu/index.js @@ -1,5 +1,5 @@ -import { app, errorHandler } from './server'; -import sparql from './sparql'; +import { app, errorHandler } from './server.js'; +import sparql from './sparql.js'; import { v1 as uuidV1 } from 'uuid'; // generates a uuid diff --git a/helpers/mu/package.json b/helpers/mu/package.json new file mode 100644 index 0000000..dd23a57 --- /dev/null +++ b/helpers/mu/package.json @@ -0,0 +1,9 @@ +{ + "name": "mu", + "version": "0.0.1", + "description": "Framework layer functions for mu-javascript-template.", + "main": "index.js", + "type": "module", + "author": "Semantic Works", + "license": "MIT" +} diff --git a/helpers/mu/sparql.js b/helpers/mu/sparql.js index 3d009b1..e56a6b7 100644 --- a/helpers/mu/sparql.js +++ b/helpers/mu/sparql.js @@ -15,23 +15,50 @@ const RETRY_TIMEOUT_INCREMENT_FACTOR = env.get('MU_QUERY_RETRY_TIMEOUT_INCREMENT //==-- logic --==// -// executes a query (you can use the template syntax) -function query( queryString, extraHeaders = {}, connectionOptions = {} ) { +/** + * Executes a SPARQL query against a given endpoint (you can use the template syntax). + * + * @param {string} queryString - The SPARQL query to execute. + * @param {object} [options={}] - Optional parameters for query execution. + * @param {string} [options.sparqlEndpoint=MU_SPARQL_ENDPOINT] - The SPARQL endpoint to send the request to. + * @param {boolean} [options.sudo=false] - Whether to include the 'mu-auth-sudo' header. + * @param {string} [options.scope] - Authentication scope to use. Falls back to DEFAULT_MU_AUTH_SCOPE if not provided. + * @param {object} [options.extraHeaders={}] - Additional headers to include in the request. + * @param {string} [options.authUser] - Username for HTTP authentication. + * @param {string} [options.authPassword] - Password for HTTP authentication. + * @param {"basic"|"digest"} [options.authType="digest"] - Type of HTTP authentication (default is digest). + * @returns {Promise} - The parsed JSON response from the SPARQL endpoint. + * @throws {Error} - Throws an error if the request fails and cannot be retried. + */ +function query( queryString, options = {} ) { if (LOG_SPARQL_QUERIES) { console.log(queryString); } - return executeQuery(queryString, extraHeaders, connectionOptions); + return executeQuery(queryString, options); }; -// executes an update query -function update(queryString, extraHeaders = {}, connectionOptions = {}) { +/** + * Executes a SPARQL query against a given endpoint (you can use the template syntax). + * + * @param {string} queryString - The SPARQL query to execute. + * @param {object} [options={}] - Optional parameters for query execution. + * @param {string} [options.sparqlEndpoint=MU_SPARQL_ENDPOINT] - The SPARQL endpoint to send the request to. + * @param {boolean} [options.sudo=false] - Whether to include the 'mu-auth-sudo' header. + * @param {string} [options.scope] - Authentication scope to use. Falls back to DEFAULT_MU_AUTH_SCOPE if not provided. + * @param {object} [options.extraHeaders={}] - Additional headers to include in the request. + * @param {string} [options.authUser] - Username for HTTP authentication. + * @param {string} [options.authPassword] - Password for HTTP authentication. + * @param {"basic"|"digest"} [options.authType="digest"] - Type of HTTP authentication (default is digest). + * @returns {Promise} - The parsed JSON response from the SPARQL endpoint. + * @throws {Error} - Throws an error if the request fails and cannot be retried. + */ +function update(queryString, options = {}) { if (LOG_SPARQL_UPDATES) { console.log(queryString); } - return executeQuery(queryString); + return executeQuery(queryString, options); }; - function defaultHeaders() { const headers = new Headers(); headers.set("content-type", "application/x-www-form-urlencoded"); @@ -46,13 +73,46 @@ function defaultHeaders() { return headers; } -async function executeQuery(queryString, extraHeaders = {}, connectionOptions = {}, attempt = 0) +/** + * Executes a SPARQL query against a given endpoint. + * + * @param {string} queryString - The SPARQL query to execute. + * @param {object} [options={}] - Optional parameters for query execution. + * @param {string} [options.sparqlEndpoint=MU_SPARQL_ENDPOINT] - The SPARQL endpoint to send the request to. + * @param {boolean} [options.sudo=false] - Whether to include the 'mu-auth-sudo' header. + * @param {string} [options.scope] - Authentication scope to use. Falls back to DEFAULT_MU_AUTH_SCOPE if not provided. + * @param {object} [options.extraHeaders={}] - Additional headers to include in the request. + * @param {string} [options.authUser] - Username for HTTP authentication. + * @param {string} [options.authPassword] - Password for HTTP authentication. + * @param {"basic"|"digest"} [options.authType="digest"] - Type of HTTP authentication (default is digest). + * @param {number} [attempt=0] - Current retry attempt. + * @returns {Promise} - The parsed JSON response from the SPARQL endpoint. + * @throws {Error} - Throws an error if the request fails and cannot be retried. + */ +async function executeQuery(queryString, options = {}, attempt = 0) { - const sparqlEndpoint = connectionOptions.sparqlEndpoint ?? MU_SPARQL_ENDPOINT; + const sparqlEndpoint = options.sparqlEndpoint ?? MU_SPARQL_ENDPOINT; const headers = defaultHeaders(); + + const extraHeaders = options.extraHeaders ?? {}; for (const key of Object.keys(extraHeaders)) { - headers.append(key, extraHeaders[key]); + headers.append(key, options.extraHeaders[key]); + } + if (options.sudo === true) { + if (env.get("ALLOW_MU_AUTH_SUDO").asBool()) { + headers.set('mu-auth-sudo', "true"); + } + else { + throw new Error("Error, sudo request but service lacks ALLOW_MU_AUTH_SUDO header"); + } } + + if (options.scope) { + headers.set('mu-auth-scope', options.scope); + } else if (env.get("DEFAULT_MU_AUTH_SCOPE")) { + headers.set('mu-auth-scope', env.get("DEFAULT_MU_AUTH_SCOPE")); + } + if (DEBUG_AUTH_HEADERS) { const stringifiedHeaders = Array.from(headers.entries()) .filter(([key]) => key.startsWith("mu-")) @@ -68,11 +128,11 @@ async function executeQuery(queryString, extraHeaders = {}, connectionOptions = headers.append("Content-Length", formData.toString().length.toString()); let response; - if (connectionOptions.authUser && connectionOptions.authPassword) { + if (options.authUser && options.authPassword) { const client = new DigestFetch( - connectionOptions.authUser, - connectionOptions.authPassword, - { basic: connectionOptions.authType === "basic" } + options.authUser, + options.authPassword, + { basic: options.authType === "basic" } ); response = await client.fetch(sparqlEndpoint, { method: "POST", @@ -93,17 +153,16 @@ async function executeQuery(queryString, extraHeaders = {}, connectionOptions = throw new Error(`HTTP Error Response: ${response.status} ${response.statusText}`); } } catch (ex) { - if (mayRetry(ex, attempt, connectionOptions)) { + if (mayRetry(ex, attempt, options)) { attempt += 1; const sleepTime = nextAttemptTimeout(attempt); console.log(`Sleeping ${sleepTime} ms before next attempt`); await new Promise((r) => setTimeout(r, sleepTime)); - return await executeRawQuery( + return await executeQuery( queryString, - extraHeaders, - connectionOptions, + options, attempt ); } else { @@ -210,10 +269,23 @@ function sparqlEscapeDate( value ){ return '"' + new Date(value).toISOString().substring(0, 10) + '"^^xsd:date'; // only keep 'YYYY-MM-DD' portion of the string }; +/** + * Escape date string or date object into an xsd:dateTime for use in a SPARQL string. + * + * @param { Date | string | number } value Date representation + * (understood by `new Date`) to convert. + * @return { string } Date representation for SPARQL query. + */ function sparqlEscapeDateTime( value ){ return '"' + new Date(value).toISOString() + '"^^xsd:dateTime'; }; +/** + * Escape boolean-like value into xsd:boolean for use in a SPARQL string. + * + * @param { any } value Boolean-like value, anything javascript finds truethy is true. + * @return { string } Boolean representation for SPARQL query. + */ function sparqlEscapeBool( value ){ return value ? '"true"^^xsd:boolean' : '"false"^^xsd:boolean'; }; @@ -251,6 +323,7 @@ const exports = { sparqlEscape: sparqlEscape, sparqlEscapeString: sparqlEscapeString, sparqlEscapeUri: sparqlEscapeUri, + sparqlEscapeDecimal: sparqlEscapeDecimal, sparqlEscapeInt: sparqlEscapeInt, sparqlEscapeFloat: sparqlEscapeFloat, sparqlEscapeDate: sparqlEscapeDate, @@ -274,4 +347,3 @@ export { sparqlEscapeDateTime, sparqlEscapeBool }; - diff --git a/merge-package-json.js b/merge-package-json.js new file mode 100644 index 0000000..cb8fb03 --- /dev/null +++ b/merge-package-json.js @@ -0,0 +1,49 @@ +import fs from 'fs'; + +// combines package.jsons and writes them to /tmp/merged-package.json +function mergePackageJson() { + const templatePackage = JSON.parse(fs.readFileSync("package.json", "utf8")); + + if (fs.existsSync("/app/package.json")) { + const servicePackage = JSON.parse( + fs.readFileSync("/app/package.json", "utf8") + ); + + servicePackage.dependencies = servicePackage.dependencies || {}; + + servicePackage.dependencies = { + ...servicePackage.dependencies, + ...templatePackage.dependencies, + }; + + warnAboutVersionDifferences(templatePackage, servicePackage); + + fs.writeFileSync( + "/tmp/merged-package.json", + JSON.stringify(servicePackage, null, 2) + ); + } else { + // just use the template package.json + fs.writeFileSync( + "/tmp/merged-package.json", + JSON.stringify(templatePackage, null, 2) + ); + } +} + +function warnAboutVersionDifferences(templatePackage, servicePackage) { + Object.keys(templatePackage.dependencies).forEach((dep) => { + if ( + servicePackage.dependencies[dep] && + templatePackage.dependencies[dep] !== servicePackage.dependencies[dep] + ) { + const winner = "template"; + // QUESTION: Can we cope with compatible definitions? + console.warn( + `Warning: Dependency ${dep} has different versions in template and service package.json. Using ${winner} version.` + ); + } + }); +} + +mergePackageJson(); diff --git a/npm-install-dependencies.sh b/npm-install-dependencies.sh new file mode 100755 index 0000000..71f25e1 --- /dev/null +++ b/npm-install-dependencies.sh @@ -0,0 +1,16 @@ +#!/bin/bash +cd /usr/src/app/app/ +if [ -f "/usr/src/app/app/package.json" ] +then + if [[ "$1" == "production" ]] && [ -f "/usr/src/app/app/package-lock.json" ] + then + echo "Installing dependencies in ci mode" + npm ci + else + echo "Installing dependencies from package.json" + npm install + fi +fi + +mkdir -p /usr/src/app/app/node_modules/ +cp -R /usr/src/app/helpers/mu /usr/src/app/app/node_modules/mu diff --git a/package.json b/package.json index f8740e2..61e3dac 100644 --- a/package.json +++ b/package.json @@ -2,6 +2,7 @@ "name": "mu-javascript-template", "version": "1.8.0", "description": "Template for mu services written in JavaScript", + "type": "module", "repository": { "type": "git", "url": "git+https://github.com/mu-semtech/mu-javascript-template.git" @@ -10,9 +11,9 @@ "mu-semtech" ], "dependencies": { + "babel-plugin-add-import-extension": "^1.6.0", "@babel/cli": "^7.20.7", "@babel/core": "^7.20.12", - "@babel/node": "^7.20.7", "@babel/plugin-proposal-decorators": "^7.22.15", "@babel/preset-env": "^7.22.10", "@babel/preset-typescript": "^7.18.6", diff --git a/prepare-package-json.sh b/prepare-package-json.sh new file mode 100755 index 0000000..0a5201e --- /dev/null +++ b/prepare-package-json.sh @@ -0,0 +1,20 @@ +#!/bin/bash +cd /usr/src/app/ +node ./merge-package-json.js +mv /tmp/merged-package.json /usr/src/app/app/package.json +cd /usr/src/app/app/ + +## Ensure package.json has module +cat /usr/src/app/app/package.json | jq -e ".type" > /dev/null +if [ $? -ne 0 ] +then + echo '[WARNING] Adding "type": "module" to your package.json.' + echo 'To remove this warning, add "type": "module" at the same level as "name" in your package.json' + sed -i '0,/{/s/{/{\n "type": "module",/' /usr/src/app/app/package.json +else + PACKAGE_TYPE=`cat /usr/src/app/app/package.json | jq -r ".type"` + if [[ "$PACKAGE_TYPE" -ne "module" ]] + then + echo '[WARNING] DIFFERENT TYPE THAN "module" IN package.json; CONTINUING WITH UNSPECIFIED BEHAVIOUR' + fi +fi diff --git a/run-development.sh b/run-development.sh index f823f79..f2f5ab1 100755 --- a/run-development.sh +++ b/run-development.sh @@ -1,5 +1,4 @@ #!/bin/bash - source ./helpers.sh # We want to run from /app but don't want to touch that folder. @@ -19,15 +18,27 @@ cd /usr/src/app/ # Install dependencies ###################### -## Check if package.json existed and did not change since previous build (/usr/src/app/app/ is copied later in this script, at first run from the template itself it doesn't exist but that's fine for comparison) -cmp -s /app/package.json /usr/src/app/app/package.json +## Check if package.json existed and did not change since previous build (at +## first run from the template itself it doesn't exist but that's fine for +## comparison) +cmp -s /app/package.json /tmp/last-build-service-package.json CHANGE_IN_PACKAGE_JSON="$?" +if [ -f /app/package.json ] +then + cp /app/package.json /tmp/last-build-service-package.json +fi ## Ensure we _sync_ the sources from the hosted app and _copy_ the node_modules. ## ## We don't want to do --delete on the node_modules because this allows us ## to depend on the node_modules installed in an earlier update cycle as well as ## taking node_modules from the hosted app into account. +## +## Although we can always override the mu package, we may install the +## node_modules in the template, or the node_modules may be offered in part or +## in full. Hence we should only add the node_modules of the mounted code and +## not remove anything. +# TODO: this is related to installing dependencies, should this become part of npm-install-dependencies or should this be part of a copy-sources script. docker-rsync --delete --exclude node_modules /app/ /usr/src/app/app/ if [ -d /app/node_modules/ ] then @@ -42,29 +53,32 @@ then fi ## Install dependencies on first boot -if [ $CHANGE_IN_PACKAGE_JSON != "0" ] && [ -f ./app/package.json ] +if [ $CHANGE_IN_PACKAGE_JSON != "0" ] || [ ! -f /tmp/dependencies-installed-once-for-dev ] then - echo "Running npm install" - cd /usr/src/app/app/ - npm install - cd /usr/src/app/ + echo "Installing dependencies" + ./prepare-package-json.sh + ./npm-install-dependencies.sh + touch /tmp/dependencies-installed-once-for-dev +else + # TODO: We overwrote the merged package.json when copying from the template, we could drop this if + # don't overwrite the merged package.json on reload. + ./prepare-package-json.sh fi - ############### # Transpilation ############### ./transpile-sources.sh - +cp /usr/src/app/helpers/mu/package.json /usr/src/dist/node_modules/mu/ ############## # Start server ############## -cd /usr/src/build/ -/usr/src/app/node_modules/.bin/babel-node \ +cd /usr/src/dist/ +node \ --inspect="0.0.0.0:9229" \ ./app.js diff --git a/run-production.sh b/run-production.sh new file mode 100755 index 0000000..248cc2d --- /dev/null +++ b/run-production.sh @@ -0,0 +1,46 @@ +#!/bin/bash + +diff -rq /app /app.original > /dev/null +APP_FILES_CHANGED="$?" +diff -rq /config /config.original > /dev/null +CONFIG_FILES_CHANGED="$?" + +if [ ! -f /usr/src/dist/app.js ] +then + echo "No built sources found. If you mount new sources, please set the NODE_ENV=\"development\" environment variable." + sleep 5; + exit 1; +elif [ $APP_FILES_CHANGED != "0" ] +then + echo "Built sources are not the same as sources available in /app. If you mount new sources, please set the NODE_ENV=\"development\" environment variable." + sleep 5; + exit 1; +elif [ $CONFIG_FILES_CHANGED != "0" ] +then + echo "Rebuilding sources to include /config." + + # move new configuration into app for transpilation + if [[ "$(ls -A /config 2> /dev/null)" ]] + then + cp -Rf /config/* /usr/src/app/app/config/ + fi + + # make a backup of the used configuration so we can detect changes + rm -Rf /config.original + mkdir /config.original + if [[ "$(ls -A /config 2> /dev/null)" ]] + then + cp -Rf /config/* /config.original + fi + + # transpile sources + cd /usr/src/app/ + ./transpile-sources.sh + + # boot transpiled sources + cd /usr/src/dist/ + exec node ./app.js +else + cd /usr/src/dist/ + exec node ./app.js +fi diff --git a/scripts/config.json b/scripts/config.json index 2e08673..84f534f 100644 --- a/scripts/config.json +++ b/scripts/config.json @@ -4,7 +4,7 @@ { "documentation": { "command": "dev-script", - "description": "Outputs a development script for this service", + "description": "Outputs a development script to use in docker-compose", "arguments": [] }, "environment": { @@ -15,6 +15,21 @@ "mounts": { "service": "/data/service" } + }, + { + "documentation": { + "command": "setup-ide", + "description": "Installs combined node modules of service and template", + "arguments": [] + }, + "environment": { + "image": "semtech/mu-javascript-template:feature-dev-experience-tryouts", + "interactive": false, + "script": "setup-ide/run.sh" + }, + "mounts": { + "service": "/app" + } } ] } diff --git a/scripts/setup-ide/run.sh b/scripts/setup-ide/run.sh new file mode 100755 index 0000000..01f66da --- /dev/null +++ b/scripts/setup-ide/run.sh @@ -0,0 +1,23 @@ +#!/bin/bash +source /usr/src/app/helpers.sh + +# mkdir -p /app +# if [ -f /data/service/package.json ] +# then +# # install script expects service package.json to be in /app +# cp /data/service/package.json /app/package.json +# fi + +# this script runs from a clean slate, the app hasn't ran yet so /usr/src/app/app is empty +docker-rsync /app /usr/src/app/app +cd /usr/src/app +./prepare-package-json.sh +./npm-install-dependencies.sh + +# copy results into the service's source folder +docker-rsync /usr/src/app/app/node_modules/ /app/node_modules +# TODO: this should be behind an ENV var and should be updated when running development so it's kept up-to-date. It may be that this requires us to have the node_modules locally but only saving in this case makes it feel a bit brittle +if [ -f /usr/src/app/app/package-lock.json ] +then + cp /usr/src/app/app/package-lock.json /app/package-lock.json +fi diff --git a/transpile-sources.sh b/transpile-sources.sh index 85d1e37..2adff56 100755 --- a/transpile-sources.sh +++ b/transpile-sources.sh @@ -1,118 +1,81 @@ #!/bin/bash - source ./helpers.sh #### #### BUILDS SOURCES #### #### Expects sources to be in /usr/src/app/ with the app in -#### /usr/src/app/app/ and stores the resulting build in /usr/src/build +#### /usr/src/app/app/ and stores the resulting build in /usr/src/dist -# Clean starting state -rm -Rf /usr/src/processing /usr/src/build +### +# PREPARE FOLDERS +### -# Copy template (/usr/src/app/) and app (/usr/src/app/app/) sources -# without package.json, which we want to skip as it would conflict -# building sources. +# NOTE: We don't clean a starting state because we can incrementally update the +# previous build -cp -R /usr/src/app /usr/src/processing -rm -f /usr/src/processing/app/package.json +# Prepare the build folder if it doesn't exist yet +mkdir -p /usr/src/dist +# Copy app from /usr/src/app/app/ sources without package.json, which we want to +# skip as it would conflict building sources. +docker-rsync --exclude node_modules --delete /usr/src/app/app/ /usr/src/dist +rm -f /usr/src/dist/package.json -## CoffeeScript -## -## Coffeescript is transpiled ready for nodejs. This is then moved into -## app so we have the javascript available which other preprocessors may -## expect to exist. +############################ +# CoffeeScript Transpilation +############################ + +## Coffeescript is transpiled ready for nodejs. This is then moved into app so +## we have the javascript available which other preprocessors may expect to +## exist. ## -## In order to generate the sourcemaps correctly, it seems we have to be -## next to the folder where we want the sources to land, but in order to -## transpile correctly we also need the node_modules for babel and the +## In order to transpile correctly we need the node_modules for babel and the ## babelrc file. We temporarily move those around. +cp /usr/src/app/babel.config.json /usr/src/dist/ -cd /usr/src/ +# make the build and move to coffeescript-transpilation `-m` for external +# sourcemaps `-M` for inlined maps. Hence -m -M will generate both. -# prepare the build folders -mkdir /usr/src/build /usr/src/build.coffee -cp -R /usr/src/processing/app/* /usr/src/build/ -cp /usr/src/processing/babel.config.json /usr/src/ -cp -R /usr/src/processing/node_modules/ /usr/src/ +# Set the NODE_PATH environment variable so we don't have to assume the +# node_modules for coffeescript transpilation are available in this folder. We +# need these for the babel plugins in babel.config.json -# make the build and move to coffeescript-transpilation -/usr/src/app/node_modules/.bin/coffee -M -m --compile -t --output ./build.coffee/ ./build -mv build.coffee/ /usr/src/processing/coffeescript-transpilation +# We transpile from . so we have a nice folder mentioned in the inlined SourceMap +pushd /usr/src/dist/ > /dev/null +NODE_PATH="/usr/src/app/node_modules/" /usr/src/app/node_modules/.bin/coffee -M --compile -t . +popd > /dev/null # clean up -rm -Rf /usr/src/build /usr/src/node_modules/ -rm /usr/src/babel.config.json +rm /usr/src/dist/babel.config.json + + +################################## +# TypeScript and ES6 transpilation +################################## ## TypeScript and ES6 ## ## Transpiles TypeScript and ES6 to something nodejs wants to run. -cd /usr/src/processing/ - -mkdir typescript-transpilation build -cp -R ./app/* build - -docker-rsync /usr/src/processing/coffeescript-transpilation/ /usr/src/processing/build/ - -/usr/src/app/node_modules/.bin/babel \ - ./build/ \ - --out-dir ./typescript-transpilation/ \ - --source-maps true \ +# +# --source-maps both would give both separate sourcemaps and inline sourcemaps +# but makes Chromium unhappy. Set to inline to only get inline sourcemaps and +# make Chromium happy. Set to true to get external sourcemaps. + +# We transpile from . so we have a nice folder mentioned in the inlined SourceMap +pushd /usr/src/dist > /dev/null +NODE_PATH="/usr/src/app/node_modules/" /usr/src/app/node_modules/.bin/babel \ + . \ + --out-dir . \ + --source-maps inline \ + --config-file "/usr/src/app/babel.config.json" \ --extensions ".ts,.js" - -rm -Rf ./build -mv typescript-transpilation /usr/src/build - -# We move the coffeescript files again because the previous step will -# have built the sources coffeescript generated, but these sources were -# already node compliant. We could make coffeescript emit ES6 and -# transpile them to nodejs in this step, but that breaks SourceMaps. -docker-rsync /usr/src/processing/coffeescript-transpilation/ /usr/src/build/ - -# We move all unhandled files (non js, ts, coffee) into the sources for -# later use. - -docker-rsync \ - --exclude "*.js" \ - --exclude "*.ts" \ - --exclude "*.coffee" \ - --exclude "node_modules/" \ - --exclude "./Dockerfile" \ - /usr/src/processing/app/ /usr/src/build/ +popd > /dev/null ############## # Node modules ############## -cd /usr/src/processing/ - -## template modules -cp -R /usr/src/processing/node_modules /usr/src/build/ - -## app modules -if [ -d /usr/src/processing/app/node_modules ] -then - docker-rsync /usr/src/processing/app/node_modules /usr/src/build/ -fi - -## mu helpers -cd /usr/src/processing/ - -mkdir /usr/src/processing/built-mu -/usr/src/app/node_modules/.bin/babel \ - /usr/src/processing/helpers/mu/ \ - --source-maps true \ - --out-dir /usr/src/processing/built-mu \ - --extensions ".js" - -cp -R /usr/src/processing/built-mu /usr/src/build/node_modules/mu - - - -## Clean temporary folders -## -## We have created garbage, let's remove it -cd /usr/src/ -rm -Rf /usr/src/processing +## merged template and app modules with mu module +docker-rsync --delete /usr/src/app/app/node_modules /usr/src/dist/ +docker-rsync /usr/src/app/app/package.json /usr/src/dist/package.json diff --git a/tsconfig-base.json b/tsconfig-base.json index e5d6c0c..35de81d 100644 --- a/tsconfig-base.json +++ b/tsconfig-base.json @@ -4,7 +4,7 @@ "compilerOptions": { "lib": ["es2020"], - "module": "commonjs", + "module": "es2020", "target": "es2020", "allowJs": true, "noEmit": true,