Re-implement Docker workflow using GitHub Actions (#326)

* Recreate a docker dir with all docker files inside and add init.sh
* Add docker/README.md with example docker-compose.xml
* Add mysql vars for mempool-config.json
* Port can be set from env var in docker-compose.xml
* Update docker-compose.xml example to use latest tag
* Remove armv7 architecture from build workflow
* Remove master git-hash file
* Remove useless ':' in sed
This commit is contained in:
Bastien Guillaumat
2021-02-10 14:51:01 +01:00
committed by GitHub
parent 54e1f38d2f
commit 5fbc952fda
14 changed files with 135 additions and 205 deletions

99
docker/README.md Normal file
View File

@@ -0,0 +1,99 @@
# Docker
## Initialization
In an empty dir create 2 sub-dirs
```bash
mkdir -p data mysql/data mysql/db-scripts
```
In the mysql/db-scripts sub-dir add the mariadb-structure.sql file from the mempool repo
Your dir should now look like that:
```bash
$ls -R
.:
data mysql
./data:
./mysql:
data db-scripts
./mysql/data:
./mysql/db-scripts:
mariadb-structure.sql
```
In the main dir add the following docker-compose.yml
```bash
version: "3.7"
services:
web:
image: mempool/frontend:latest
user: "1000:1000"
restart: on-failure
stop_grace_period: 1m
command: "./wait-for db:3306 --timeout=720 -- nginx -g 'daemon off;'"
ports:
- 80:8080
environment:
FRONTEND_HTTP_PORT: "8080"
BACKEND_MAINNET_HTTP_HOST: "api"
api:
image: mempool/backend:v2.1.0
user: "1000:1000"
restart: on-failure
stop_grace_period: 1m
command: "./wait-for-it.sh db:3306 --timeout=720 --strict -- ./start.sh"
volumes:
- ./data:/backend/cache
environment:
RPC_HOST: "127.0.0.1"
RPC_PORT: "8332"
RPC_USER: "mempool"
RPC_PASS: "mempool"
ELECTRS_HOST: "127.0.0.1"
ELECTRS_PORT: "50002"
MYSQL_HOST: "db"
MYSQL_PORT: "3306"
MYSQL_DATABASE: "mempool"
MYSQL_USER: "mempool"
MYSQL_PASS: "mempool"
BACKEND_MAINNET_HTTP_PORT: "8999"
CACHE_DIR: "/backend/cache/"
db:
image: mariadb:10.5.8
user: "1000:1000"
restart: on-failure
stop_grace_period: 1m
volumes:
- ./mysql/data:/var/lib/mysql
- ./mysql/db-scripts:/docker-entrypoint-initdb.d
environment:
MYSQL_DATABASE: "mempool"
MYSQL_USER: "mempool"
MYSQL_PASSWORD: "mempool"
MYSQL_ROOT_PASSWORD: "admin"
```
You can update all the environment variables inside the API container, especially the RPC and ELECTRS ones
## Run it
To run our docker-compose use the following cmd:
```bash
docker-compose up
```
If everything went okay you should see the beautiful mempool :grin:
If you get stuck on "loading blocks", this means the websocket can't connect.
Check your nginx proxy setup, firewalls, etc. and open an issue if you need help.

27
docker/backend/Dockerfile Normal file
View File

@@ -0,0 +1,27 @@
FROM node:12-buster-slim AS builder
WORKDIR /build
COPY . .
RUN apt-get update
RUN apt-get install -y build-essential python3 pkg-config
RUN npm ci --production
RUN npm i typescript
RUN npm run build
FROM node:12-buster-slim
WORKDIR /backend
COPY --from=builder /build/ .
RUN chmod +x /backend/start.sh
RUN chmod +x /backend/wait-for-it.sh
RUN chown -R 1000:1000 /backend && chmod -R 755 /backend
USER 1000
EXPOSE 8999
CMD ["/backend/start.sh"]

View File

@@ -0,0 +1,38 @@
{
"MEMPOOL": {
"NETWORK": "mainnet",
"BACKEND": "electrum",
"HTTP_PORT": __MEMPOOL_BACKEND_MAINNET_HTTP_PORT__,
"SPAWN_CLUSTER_PROCS": 0,
"API_URL_PREFIX": "/api/v1/",
"POLL_RATE_MS": 2000,
"CACHE_DIR": "__MEMPOOL_BACKEND_MAINNET_CACHE_DIR__"
},
"CORE_RPC": {
"HOST": "__BITCOIN_MAINNET_RPC_HOST__",
"PORT": __BITCOIN_MAINNET_RPC_PORT__,
"USERNAME": "__BITCOIN_MAINNET_RPC_USER__",
"PASSWORD": "__BITCOIN_MAINNET_RPC_PASS__"
},
"ELECTRUM": {
"HOST": "__ELECTRS_MAINNET_HTTP_HOST__",
"PORT": __ELECTRS_MAINNET_HTTP_PORT__,
"TLS_ENABLED": false,
"TX_LOOKUPS": true
},
"ESPLORA": {
"REST_API_URL": "http://127.0.0.1:3000"
},
"DATABASE": {
"ENABLED": true,
"HOST": "__MYSQL_HOST__",
"PORT": __MYSQL_PORT__,
"DATABASE": "__MYSQL_DATABASE__",
"USERNAME": "__MYSQL_USERNAME__",
"PASSWORD": "__MYSQL_PASSWORD__"
},
"STATISTICS": {
"ENABLED": true,
"TX_PER_SECOND_SAMPLE_PERIOD": 150
}
}

37
docker/backend/start.sh Normal file
View File

@@ -0,0 +1,37 @@
#!/bin/sh
#MEMPOOL
__MEMPOOL_BACKEND_MAINNET_HTTP_PORT__=${BACKEND_MAINNET_HTTP_PORT:=8999}
__MEMPOOL_BACKEND_MAINNET_CACHE_DIR__=${CACHE_DIR:=./}
# BITCOIN
__BITCOIN_MAINNET_RPC_HOST__=${RPC_HOST:=127.0.0.1}
__BITCOIN_MAINNET_RPC_PORT__=${RPC_PORT:=8332}
__BITCOIN_MAINNET_RPC_USER__=${RPC_USER:=mempool}
__BITCOIN_MAINNET_RPC_PASS__=${RPC_PASS:=mempool}
# ELECTRUM
__ELECTRS_MAINNET_HTTP_HOST__=${ELECTRS_HOST:=127.0.0.1}
__ELECTRS_MAINNET_HTTP_PORT__=${ELECTRS_PORT:=50002}
# MYSQL
__MYSQL_HOST__=${MYSQL_HOST:=127.0.0.1}
__MYSQL_PORT__=${MYSQL_PORT:=3306}
__MYSQL_DATABASE__=${MYSQL_DATABASE:=mempool}
__MYSQL_USERNAME__=${MYSQL_USER:=mempool}
__MYSQL_PASSWORD__=${MYSQL_PASS:=mempool}
mkdir -p "${__MEMPOOL_BACKEND_MAINNET_CACHE_DIR__}"
sed -i "s/__BITCOIN_MAINNET_RPC_HOST__/${__BITCOIN_MAINNET_RPC_HOST__}/g" mempool-config.json
sed -i "s/__BITCOIN_MAINNET_RPC_PORT__/${__BITCOIN_MAINNET_RPC_PORT__}/g" mempool-config.json
sed -i "s/__BITCOIN_MAINNET_RPC_USER__/${__BITCOIN_MAINNET_RPC_USER__}/g" mempool-config.json
sed -i "s/__BITCOIN_MAINNET_RPC_PASS__/${__BITCOIN_MAINNET_RPC_PASS__}/g" mempool-config.json
sed -i "s/__ELECTRS_MAINNET_HTTP_HOST__/${__ELECTRS_MAINNET_HTTP_HOST__}/g" mempool-config.json
sed -i "s/__ELECTRS_MAINNET_HTTP_PORT__/${__ELECTRS_MAINNET_HTTP_PORT__}/g" mempool-config.json
sed -i "s/__MYSQL_HOST__/${__MYSQL_HOST__}/g" mempool-config.json
sed -i "s/__MYSQL_PORT__/${__MYSQL_PORT__}/g" mempool-config.json
sed -i "s/__MYSQL_DATABASE__/${__MYSQL_DATABASE__}/g" mempool-config.json
sed -i "s/__MYSQL_USERNAME__/${__MYSQL_USERNAME__}/g" mempool-config.json
sed -i "s/__MYSQL_PASSWORD__/${__MYSQL_PASSWORD__}/g" mempool-config.json
sed -i "s!__MEMPOOL_BACKEND_MAINNET_CACHE_DIR__!${__MEMPOOL_BACKEND_MAINNET_CACHE_DIR__}!g" mempool-config.json
sed -i "s/__MEMPOOL_BACKEND_MAINNET_HTTP_PORT__/${__MEMPOOL_BACKEND_MAINNET_HTTP_PORT__}/g" mempool-config.json
node /backend/dist/index.js

View File

@@ -0,0 +1,182 @@
#!/usr/bin/env bash
# Use this script to test if a given TCP host/port are available
WAITFORIT_cmdname=${0##*/}
echoerr() { if [[ $WAITFORIT_QUIET -ne 1 ]]; then echo "$@" 1>&2; fi }
usage()
{
cat << USAGE >&2
Usage:
$WAITFORIT_cmdname host:port [-s] [-t timeout] [-- command args]
-h HOST | --host=HOST Host or IP under test
-p PORT | --port=PORT TCP port under test
Alternatively, you specify the host and port as host:port
-s | --strict Only execute subcommand if the test succeeds
-q | --quiet Don't output any status messages
-t TIMEOUT | --timeout=TIMEOUT
Timeout in seconds, zero for no timeout
-- COMMAND ARGS Execute command with args after the test finishes
USAGE
exit 1
}
wait_for()
{
if [[ $WAITFORIT_TIMEOUT -gt 0 ]]; then
echoerr "$WAITFORIT_cmdname: waiting $WAITFORIT_TIMEOUT seconds for $WAITFORIT_HOST:$WAITFORIT_PORT"
else
echoerr "$WAITFORIT_cmdname: waiting for $WAITFORIT_HOST:$WAITFORIT_PORT without a timeout"
fi
WAITFORIT_start_ts=$(date +%s)
while :
do
if [[ $WAITFORIT_ISBUSY -eq 1 ]]; then
nc -z $WAITFORIT_HOST $WAITFORIT_PORT
WAITFORIT_result=$?
else
(echo -n > /dev/tcp/$WAITFORIT_HOST/$WAITFORIT_PORT) >/dev/null 2>&1
WAITFORIT_result=$?
fi
if [[ $WAITFORIT_result -eq 0 ]]; then
WAITFORIT_end_ts=$(date +%s)
echoerr "$WAITFORIT_cmdname: $WAITFORIT_HOST:$WAITFORIT_PORT is available after $((WAITFORIT_end_ts - WAITFORIT_start_ts)) seconds"
break
fi
sleep 1
done
return $WAITFORIT_result
}
wait_for_wrapper()
{
# In order to support SIGINT during timeout: http://unix.stackexchange.com/a/57692
if [[ $WAITFORIT_QUIET -eq 1 ]]; then
timeout $WAITFORIT_BUSYTIMEFLAG $WAITFORIT_TIMEOUT $0 --quiet --child --host=$WAITFORIT_HOST --port=$WAITFORIT_PORT --timeout=$WAITFORIT_TIMEOUT &
else
timeout $WAITFORIT_BUSYTIMEFLAG $WAITFORIT_TIMEOUT $0 --child --host=$WAITFORIT_HOST --port=$WAITFORIT_PORT --timeout=$WAITFORIT_TIMEOUT &
fi
WAITFORIT_PID=$!
trap "kill -INT -$WAITFORIT_PID" INT
wait $WAITFORIT_PID
WAITFORIT_RESULT=$?
if [[ $WAITFORIT_RESULT -ne 0 ]]; then
echoerr "$WAITFORIT_cmdname: timeout occurred after waiting $WAITFORIT_TIMEOUT seconds for $WAITFORIT_HOST:$WAITFORIT_PORT"
fi
return $WAITFORIT_RESULT
}
# process arguments
while [[ $# -gt 0 ]]
do
case "$1" in
*:* )
WAITFORIT_hostport=(${1//:/ })
WAITFORIT_HOST=${WAITFORIT_hostport[0]}
WAITFORIT_PORT=${WAITFORIT_hostport[1]}
shift 1
;;
--child)
WAITFORIT_CHILD=1
shift 1
;;
-q | --quiet)
WAITFORIT_QUIET=1
shift 1
;;
-s | --strict)
WAITFORIT_STRICT=1
shift 1
;;
-h)
WAITFORIT_HOST="$2"
if [[ $WAITFORIT_HOST == "" ]]; then break; fi
shift 2
;;
--host=*)
WAITFORIT_HOST="${1#*=}"
shift 1
;;
-p)
WAITFORIT_PORT="$2"
if [[ $WAITFORIT_PORT == "" ]]; then break; fi
shift 2
;;
--port=*)
WAITFORIT_PORT="${1#*=}"
shift 1
;;
-t)
WAITFORIT_TIMEOUT="$2"
if [[ $WAITFORIT_TIMEOUT == "" ]]; then break; fi
shift 2
;;
--timeout=*)
WAITFORIT_TIMEOUT="${1#*=}"
shift 1
;;
--)
shift
WAITFORIT_CLI=("$@")
break
;;
--help)
usage
;;
*)
echoerr "Unknown argument: $1"
usage
;;
esac
done
if [[ "$WAITFORIT_HOST" == "" || "$WAITFORIT_PORT" == "" ]]; then
echoerr "Error: you need to provide a host and port to test."
usage
fi
WAITFORIT_TIMEOUT=${WAITFORIT_TIMEOUT:-15}
WAITFORIT_STRICT=${WAITFORIT_STRICT:-0}
WAITFORIT_CHILD=${WAITFORIT_CHILD:-0}
WAITFORIT_QUIET=${WAITFORIT_QUIET:-0}
# Check to see if timeout is from busybox?
WAITFORIT_TIMEOUT_PATH=$(type -p timeout)
WAITFORIT_TIMEOUT_PATH=$(realpath $WAITFORIT_TIMEOUT_PATH 2>/dev/null || readlink -f $WAITFORIT_TIMEOUT_PATH)
WAITFORIT_BUSYTIMEFLAG=""
if [[ $WAITFORIT_TIMEOUT_PATH =~ "busybox" ]]; then
WAITFORIT_ISBUSY=1
# Check if busybox timeout uses -t flag
# (recent Alpine versions don't support -t anymore)
if timeout &>/dev/stdout | grep -q -e '-t '; then
WAITFORIT_BUSYTIMEFLAG="-t"
fi
else
WAITFORIT_ISBUSY=0
fi
if [[ $WAITFORIT_CHILD -gt 0 ]]; then
wait_for
WAITFORIT_RESULT=$?
exit $WAITFORIT_RESULT
else
if [[ $WAITFORIT_TIMEOUT -gt 0 ]]; then
wait_for_wrapper
WAITFORIT_RESULT=$?
else
wait_for
WAITFORIT_RESULT=$?
fi
fi
if [[ $WAITFORIT_CLI != "" ]]; then
if [[ $WAITFORIT_RESULT -ne 0 && $WAITFORIT_STRICT -eq 1 ]]; then
echoerr "$WAITFORIT_cmdname: strict mode, refusing to execute subprocess"
exit $WAITFORIT_RESULT
fi
exec "${WAITFORIT_CLI[@]}"
else
exit $WAITFORIT_RESULT
fi

View File

@@ -0,0 +1,34 @@
FROM node:12-buster-slim AS builder
WORKDIR /build
COPY . .
RUN apt-get update
RUN apt-get install -y build-essential rsync
RUN npm i
RUN npm run build
FROM nginx:1.17.8-alpine
WORKDIR /patch
COPY --from=builder /build/entrypoint.sh .
COPY --from=builder /build/wait-for .
COPY --from=builder /build/dist/mempool /var/www/mempool
COPY --from=builder /build/nginx.conf /etc/nginx/
COPY --from=builder /build/nginx-mempool.conf /etc/nginx/conf.d/
RUN chmod +x /patch/entrypoint.sh
RUN chmod +x /patch/wait-for
RUN chown -R 1000:1000 /patch && chmod -R 755 /patch && \
chown -R 1000:1000 /var/cache/nginx && \
chown -R 1000:1000 /var/log/nginx && \
chown -R 1000:1000 /etc/nginx/nginx.conf && \
chown -R 1000:1000 /etc/nginx/conf.d
RUN touch /var/run/nginx.pid && \
chown -R 1000:1000 /var/run/nginx.pid
USER 1000
ENTRYPOINT ["/patch/entrypoint.sh"]
CMD ["nginx", "-g", "daemon off;"]

View File

@@ -0,0 +1,13 @@
#!/bin/sh
__MEMPOOL_BACKEND_MAINNET_HTTP_HOST__=${BACKEND_MAINNET_HTTP_HOST:=127.0.0.1}
__MEMPOOL_BACKEND_MAINNET_HTTP_PORT__=${BACKEND_MAINNET_HTTP_PORT:=8999}
__MEMPOOL_FRONTEND_HTTP_PORT__=${FRONTEND_HTTP_PORT:=8080}
sed -i "s/__MEMPOOL_BACKEND_MAINNET_HTTP_HOST__/${__MEMPOOL_BACKEND_MAINNET_HTTP_HOST__}/g" /etc/nginx/conf.d/nginx-mempool.conf
sed -i "s/__MEMPOOL_BACKEND_MAINNET_HTTP_PORT__/${__MEMPOOL_BACKEND_MAINNET_HTTP_PORT__}/g" /etc/nginx/conf.d/nginx-mempool.conf
cp /etc/nginx/nginx.conf /patch/nginx.conf
sed -i "s/__MEMPOOL_FRONTEND_HTTP_PORT__/${__MEMPOOL_FRONTEND_HTTP_PORT__}/g" /patch/nginx.conf
cat /patch/nginx.conf > /etc/nginx/nginx.conf
exec "$@"

84
docker/frontend/wait-for Normal file
View File

@@ -0,0 +1,84 @@
#!/bin/sh
TIMEOUT=15
QUIET=0
echoerr() {
if [ "$QUIET" -ne 1 ]; then printf "%s\n" "$*" 1>&2; fi
}
usage() {
exitcode="$1"
cat << USAGE >&2
Usage:
$cmdname host:port [-t timeout] [-- command args]
-q | --quiet Do not output any status messages
-t TIMEOUT | --timeout=timeout Timeout in seconds, zero for no timeout
-- COMMAND ARGS Execute command with args after the test finishes
USAGE
exit "$exitcode"
}
wait_for() {
if ! command -v nc >/dev/null; then
echoerr 'nc command is missing!'
exit 1
fi
for i in `seq $TIMEOUT` ; do
nc -z "$HOST" "$PORT" > /dev/null 2>&1
result=$?
if [ $result -eq 0 ] ; then
if [ $# -gt 0 ] ; then
exec "$@"
fi
exit 0
fi
sleep 1
done
echo "Operation timed out" >&2
exit 1
}
while [ $# -gt 0 ]
do
case "$1" in
*:* )
HOST=$(printf "%s\n" "$1"| cut -d : -f 1)
PORT=$(printf "%s\n" "$1"| cut -d : -f 2)
shift 1
;;
-q | --quiet)
QUIET=1
shift 1
;;
-t)
TIMEOUT="$2"
if [ "$TIMEOUT" = "" ]; then break; fi
shift 2
;;
--timeout=*)
TIMEOUT="${1#*=}"
shift 1
;;
--)
shift
break
;;
--help)
usage 0
;;
*)
echoerr "Unknown argument: $1"
usage 1
;;
esac
done
if [ "$HOST" = "" -o "$PORT" = "" ]; then
echoerr "Error: you need to provide a host and port to test."
usage 2
fi
wait_for "$@"

18
docker/init.sh Executable file
View File

@@ -0,0 +1,18 @@
#!/bin/sh
#backend
gitMaster="\.\.\/\.git\/refs\/heads\/master"
git ls-remote https://github.com/mempool/mempool.git $1 | awk '{ print $1}' > ./backend/master
cp ./docker/backend/* ./backend/
sed -i "s/${gitMaster}/master/g" ./backend/src/api/backend-info.ts
#frontend
localhostIP="127.0.0.1"
cp ./docker/frontend/* ./frontend
cp ./nginx.conf ./frontend/
cp ./nginx-mempool.conf ./frontend/
sed -i "s/${localhostIP}:80/0.0.0.0:__MEMPOOL_FRONTEND_HTTP_PORT__/g" ./frontend/nginx.conf
sed -i "s/${localhostIP}/0.0.0.0/g" ./frontend/nginx.conf
sed -i "s/user nobody;//g" ./frontend/nginx.conf
sed -i "s!/etc/nginx/nginx-mempool.conf!/etc/nginx/conf.d/nginx-mempool.conf!g" ./frontend/nginx.conf
sed -i "s/${localhostIP}:8999/__MEMPOOL_BACKEND_MAINNET_HTTP_HOST__:__MEMPOOL_BACKEND_MAINNET_HTTP_PORT__/g" ./frontend/nginx-mempool.conf