diff --git a/.env.example b/.env.example index eea77f8..e9a2301 100644 --- a/.env.example +++ b/.env.example @@ -10,6 +10,8 @@ credentials_PGPASSWORD=rootpass # auth auth_ELIXIR_ID=XC56EL11xx auth_ELIXIR_SECRET=wHPVQaYXmdDHg +auth_DB_PASSWORD=auth +auth_DB_USER=auth # rabbitmq rabbitmq_MQ_PASSWORD=test diff --git a/docker-compose-demo.yml b/docker-compose-demo.yml index 0f08651..ab81495 100644 --- a/docker-compose-demo.yml +++ b/docker-compose-demo.yml @@ -12,6 +12,8 @@ services: depends_on: s3inbox: condition: service_started + oidc: + condition: service_healthy image: python:3.10-alpine networks: - secure @@ -41,6 +43,7 @@ services: image: python:3.10-slim networks: - public + - secure ports: - "8080:8080" restart: always diff --git a/docker-compose.yml b/docker-compose.yml index f89519f..be24c33 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -27,10 +27,14 @@ services: auth: container_name: auth command: sda-auth - image: ghcr.io/neicnordic/sensitive-data-archive:v0.3.25 + image: ghcr.io/neicnordic/sensitive-data-archive:PR1099 + networks: + - secure depends_on: credentials: condition: service_completed_successfully + postgres: + condition: service_healthy environment: - OIDC_ID=${auth_ELIXIR_ID} - OIDC_PROVIDER=http://${DOCKERHOST:-localhost}:8080/oidc/ @@ -39,6 +43,8 @@ services: - OIDC_REDIRECTURL=http://localhost:8085/oidc/login - LOG_LEVEL=debug - RESIGNJWT=false + - DB_PASSWORD=${auth_DB_PASSWORD} + - DB_USER=${auth_DB_USER} extra_hosts: - ${DOCKERHOST:-localhost}:host-gateway - "host.docker.internal:host-gateway" @@ -65,7 +71,7 @@ services: interval: 5s timeout: 20s retries: 20 - image: ghcr.io/neicnordic/sensitive-data-archive:v0.3.25-rabbitmq + image: ghcr.io/neicnordic/sensitive-data-archive:PR1099-rabbitmq networks: - secure ports: @@ -89,7 +95,7 @@ services: interval: 5s timeout: 20s retries: 20 - image: ghcr.io/neicnordic/sensitive-data-archive:v0.3.25-postgres + image: ghcr.io/neicnordic/sensitive-data-archive:PR1099-postgres networks: - secure restart: always @@ -139,10 +145,9 @@ services: - DB_PASSWORD=${download_DB_PASSWORD} - DB_USER=${download_DB_USER} - OIDC_CONFIGURATION_URL=http://${DOCKERHOST:-dockerhost}:8080/oidc/.well-known/openid-configuration - - ARCHIVE_TYPE=s3seekable extra_hosts: - ${DOCKERHOST:-dockerhost}:host-gateway - image: harbor.nbis.se/gdi/sda-download:20240415 + image: ghcr.io/neicnordic/sensitive-data-archive:PR1099-download networks: - public - secure @@ -173,7 +178,7 @@ services: - BROKER_USER=${finalize_BROKER_USER} - DB_PASSWORD=${finalize_DB_PASSWORD} - DB_USER=${finalize_DB_USER} - image: ghcr.io/neicnordic/sensitive-data-archive:v0.3.25 + image: ghcr.io/neicnordic/sensitive-data-archive:PR1099 networks: - secure restart: always @@ -199,7 +204,7 @@ services: - BROKER_USER=${ingest_BROKER_USER} - DB_PASSWORD=${ingest_DB_PASSWORD} - DB_USER=${ingest_DB_USER} - image: ghcr.io/neicnordic/sensitive-data-archive:v0.3.25 + image: ghcr.io/neicnordic/sensitive-data-archive:PR1099 networks: - secure restart: always @@ -225,7 +230,7 @@ services: - BROKER_USER=${mapper_BROKER_USER} - DB_PASSWORD=${mapper_DB_PASSWORD} - DB_USER=${mapper_DB_USER} - image: ghcr.io/neicnordic/sensitive-data-archive:v0.3.25 + image: ghcr.io/neicnordic/sensitive-data-archive:PR1099 networks: - secure restart: always @@ -251,7 +256,7 @@ services: - BROKER_USER=${verify_BROKER_USER} - DB_PASSWORD=${verify_DB_PASSWORD} - DB_USER=${verify_DB_USER} - image: ghcr.io/neicnordic/sensitive-data-archive:v0.3.25 + image: ghcr.io/neicnordic/sensitive-data-archive:PR1099 networks: - secure restart: always @@ -280,7 +285,7 @@ services: - SERVER_JWTPUBKEYURL=http://${DOCKERHOST:-dockerhost}:8080/oidc/jwk extra_hosts: - ${DOCKERHOST:-dockerhost}:host-gateway - image: ghcr.io/neicnordic/sensitive-data-archive:v0.3.25 + image: ghcr.io/neicnordic/sensitive-data-archive:PR1099 networks: - public - secure @@ -297,7 +302,7 @@ services: depends_on: credentials: condition: service_completed_successfully - image: ghcr.io/neicnordic/sensitive-data-archive:v0.3.25 + image: ghcr.io/neicnordic/sensitive-data-archive:PR1099 networks: - secure restart: always diff --git a/scripts/load_data.sh b/scripts/load_data.sh index 6b96a4c..b402041 100644 --- a/scripts/load_data.sh +++ b/scripts/load_data.sh @@ -1,10 +1,17 @@ #!/bin/sh set -e -apk -q --no-cache add curl jq +apk -q --no-cache add curl jq postgresql-client pip -q install s3cmd +token=$(curl -s -k https://oidc:8080/tokens | jq -r '.[0]') + + +hash=$(sed -n '2p' /shared/c4gh.pub.pem | base64 -d -w0 | xxd -c64 -p) +# insert a c4gh public key hash +PGPASSWORD=rootpass psql -U postgres -h postgres -d sda -At -c "INSERT INTO sda.encryption_keys(key_hash, description) VALUES('$hash', 'test key') ON CONFLICT DO NOTHING;" + FILES="htsnexus_test_NA12878.bam htsnexus_test_NA12878.bam.bai htsnexus_test_NA12878.bam.blocks.yaml htsnexus_test_NA12878.bam.gzi" for file in ${FILES}; do curl -s -L -o "$file" "https://github.com/umccr/htsget-rs/raw/main/data/bam/$file" @@ -44,7 +51,7 @@ for file in ${FILES}; do jq -r -c -n \ --arg type ingest \ --arg user dummy@gdi.eu \ - --arg filepath dummy_gdi.eu/"$file.c4gh" \ + --arg filepath "$file.c4gh" \ --argjson encrypted_checksums "$encrypted_checksums" \ '$ARGS.named|@base64' ) @@ -107,7 +114,7 @@ for file in ${FILES}; do jq -r -c -n \ --arg type accession \ --arg user dummy@gdi.eu \ - --arg filepath dummy_gdi.eu/"$file.c4gh" \ + --arg filepath "$file.c4gh" \ --arg accession_id "FILE000000$I" \ --argjson decrypted_checksums "$(echo "$MSG"| jq -r '.payload|fromjson|.decrypted_checksums|tostring')" \ '$ARGS.named|@base64' diff --git a/scripts/make_credentials.sh b/scripts/make_credentials.sh index 4aadfa5..10e8580 100644 --- a/scripts/make_credentials.sh +++ b/scripts/make_credentials.sh @@ -10,7 +10,7 @@ apt-get -o DPkg::Lock::Timeout=60 install -y curl jq postgresql-client openssl > pip install --upgrade pip > /dev/null pip install aiohttp Authlib joserfc requests > /dev/null -for n in download finalize inbox ingest mapper sync verify; do +for n in api auth download finalize inbox ingest mapper sync verify; do echo "creating credentials for: $n" db_password=$(eval echo \$$n"_DB_PASSWORD") mq_password=$(eval echo \$$n"_BROKER_PASSWORD")