Synck fork 8

.github/integration/scripts/make_db_credentials.sh

@@ -4,7 +4,7 @@ set -e
 apt-get -o DPkg::Lock::Timeout=60 update > /dev/null
 apt-get -o DPkg::Lock::Timeout=60 install -y postgresql-client >/dev/null
 
-for n in api auth download finalize inbox ingest mapper sync verify; do
+for n in api auth download finalize inbox ingest mapper rotatekey sync verify; do
     echo "creating credentials for: $n"
     psql -U postgres -h migrate -d sda -c "ALTER ROLE $n LOGIN PASSWORD '$n';"
     psql -U postgres -h postgres -d sda -c "ALTER ROLE $n LOGIN PASSWORD '$n';"

.github/integration/scripts/make_sda_credentials.sh

@@ -9,12 +9,12 @@ if [ -n "$PGSSLCERT" ]; then
 fi
 
 apt-get -o DPkg::Lock::Timeout=60 update > /dev/null
-apt-get -o DPkg::Lock::Timeout=60 install -y curl jq openssh-client openssl postgresql-client >/dev/null
+apt-get -o DPkg::Lock::Timeout=60 install -y curl jq openssh-client openssl postgresql-client xxd >/dev/null
 
 pip install --upgrade pip > /dev/null
 pip install aiohttp Authlib joserfc requests > /dev/null
 
-for n in api auth download finalize inbox ingest mapper sync verify; do
+for n in api auth download finalize inbox ingest mapper rotatekey sync verify; do
     echo "creating credentials for: $n"
     psql -U postgres -h postgres -d sda -c "ALTER ROLE $n LOGIN PASSWORD '$n';"
     psql -U postgres -h postgres -d sda -c "GRANT base TO $n;"
@@ -106,11 +106,6 @@ if [ ! -f "/shared/c4gh.sec.pem" ]; then
     /shared/crypt4gh generate -n /shared/c4gh -p c4ghpass
 fi
 
-if [ ! -f "/shared/c4gh1.sec.pem" ]; then
-    echo "creating crypth4gh key"
-    /shared/crypt4gh generate -n /shared/c4gh1 -p c4ghpass
-fi
-
 if [ ! -f "/shared/client.sec.pem" ]; then # client key for re-encryption
     echo "creating client crypth4gh key"
     /shared/crypt4gh generate -n /shared/client -p c4ghpass
@@ -121,6 +116,22 @@ if [ ! -f "/shared/sync.sec.pem" ]; then
     /shared/crypt4gh generate -n /shared/sync -p syncPass
 fi
 
+if [ ! -f "/shared/rotatekey.sec.pem" ]; then
+    echo "creating rotatekey crypth4gh key"
+    /shared/crypt4gh generate -n /shared/rotatekey -p rotatekeyPass
+fi
+
+# register the rotation key in the db
+resp=$(psql -U postgres -h postgres -d sda -At -c "SELECT description FROM sda.encryption_keys;")
+if ! echo "$resp" | grep -q 'this is the new key to rotate to'; then
+    rotateKeyHash=$(cat /shared/rotatekey.pub.pem | awk 'NR==2' | base64 -d | xxd -p -c256)
+    resp=$(psql -U postgres -h postgres -d sda -At -c "INSERT INTO sda.encryption_keys(key_hash, description) VALUES('$rotateKeyHash', 'this is the new key to rotate to');")
+    if [ "$(echo "$resp" | tr -d '\n')" != "INSERT 0 1" ]; then
+        echo "insert keyhash failed"
+        exit 1
+    fi
+fi
+
 if [ ! -f "/shared/keys/ssh" ]; then
     ssh-keygen -o -a 256 -t ed25519 -f /shared/keys/ssh -N ""
     pubKey="$(cat /shared/keys/ssh.pub)"

.github/integration/sda-s3-integration.yml

@@ -11,7 +11,7 @@ services:
         condition: service_healthy
     environment:
       - PGPASSWORD=rootpasswd
-    image: python:3.11-slim
+    image: python:3.11-slim-bookworm
     volumes:
       - ./scripts:/scripts
       - shared:/shared
@@ -253,6 +253,29 @@ services:
       - ./sda/config.yaml:/config.yaml
       - shared:/shared
 
+  rotatekey:
+    image: ghcr.io/neicnordic/sensitive-data-archive:PR${PR_NUMBER}
+    command: [sda-rotatekey]
+    container_name: rotatekey
+    depends_on:
+      credentials:
+        condition: service_completed_successfully
+      postgres:
+        condition: service_healthy
+      rabbitmq:
+        condition: service_healthy
+    environment:
+      - BROKER_PASSWORD=rotatekey
+      - BROKER_USER=rotatekey
+      - BROKER_QUEUE=rotatekey
+      - BROKER_ROUTINGKEY=rotatekey
+      - DB_PASSWORD=rotatekey
+      - DB_USER=rotatekey
+    restart: always
+    volumes:
+      - ./sda/config.yaml:/config.yaml
+      - shared:/shared
+
   cega-nss:
     container_name: cega-nss
     depends_on:
@@ -384,6 +407,8 @@ services:
         condition: service_started
       reencrypt:
         condition: service_started
+      rotatekey:
+        condition: service_started
     extra_hosts:
       - "localhost:host-gateway"
     environment:

.github/integration/sda/config.yaml

@@ -74,11 +74,12 @@ c4gh:
   filePath: /shared/c4gh.sec.pem
   passphrase: "c4ghpass"
   syncPubKeyPath: /shared/sync.pub.pem
+  rotatePubKeyPath: /shared/rotatekey.pub.pem
   privateKeys:
   - filePath: /shared/c4gh.sec.pem
     passphrase: "c4ghpass"
-  - filePath: /shared/c4gh1.sec.pem
-    passphrase: "c4ghpass"
+  - filePath: /shared/rotatekey.sec.pem
+    passphrase: "rotatekeyPass"
 
 oidc:
   id: XC56EL11xx

.github/integration/sda/rbac.json

@@ -74,4 +74,4 @@
         "rolebinding": "admin"
       }
     ]
- }
+ }

.github/integration/tests/postgres/10_sanity_check.sh

@@ -22,7 +22,7 @@ if [ "$status" -eq 0 ]; then
 fi
 
 ## verify that migrations worked
-migratedb=$(find /migratedb.d/ -name "*.sql" -printf '%f\n' |  sort -n | tail -1 | cut -d '.' -f1)
+migratedb=$(find /migratedb.d/ -name "*.sql" -printf '%f\n' |  sort -n | tail -1 | cut -d '.' -f1 | cut -d '_' -f1)
 version=$(psql -U postgres -h migrate -d sda -At -c "select max(version) from sda.dbschema_version;")
 if [ "$version" -ne "$migratedb" ]; then
     echo "Migration scripts failed"

.github/integration/tests/postgres/20_inbox_queries.sh

@@ -1,16 +1,16 @@
 #!/bin/sh
 set -eou pipefail
-
+fileID="33d29907-c565-4a90-98b4-e31b992ab376"
 export PGPASSWORD=inbox
 
 for host in migrate postgres; do
-    fileID=$(psql -U inbox -h "$host" -d sda -At -c "SELECT sda.register_file('inbox/test-file.c4gh', 'test-user');")
+    fileID=$(psql -U inbox -h "$host" -d sda -At -c "SELECT sda.register_file('$fileID', 'inbox/test-file.c4gh', 'test-user');")
     if [ -z "$fileID" ]; then
         echo "register_file failed"
         exit 1
     fi
 
-    newFileID=$(psql -U inbox -h "$host" -d sda -At -c "SELECT sda.register_file('inbox/test-file.c4gh', 'other-user');")
+    newFileID=$(psql -U inbox -h "$host" -d sda -At -c "SELECT sda.register_file(null, 'inbox/test-file.c4gh', 'other-user');")
     if [ -z "$newFileID" ]; then
         echo "register_file failed"
         exit 1

.github/integration/tests/postgres/30_ingest_queries.sh

@@ -3,17 +3,17 @@ set -eou pipefail
 
 export PGPASSWORD=ingest
 user="test-user"
-corrID="33d29907-c565-4a90-98b4-e31b992ab376"
+fileID="33d29907-c565-4a90-98b4-e31b992ab376"
 
 for host in migrate postgres; do
     ## insert file
-    fileID=$(psql -U ingest -h "$host" -d sda -At -c "SELECT sda.register_file('inbox/test-file.c4gh', '$user');")
+    fileID=$(psql -U ingest -h "$host" -d sda -At -c "SELECT sda.register_file('$fileID', 'inbox/test-file.c4gh', '$user');")
     if [ -z "$fileID" ]; then
         echo "register_file failed"
         exit 1
     fi
 
-    resp=$(psql -U ingest -h "$host" -d sda -At -c "INSERT INTO sda.file_event_log(file_id, event, correlation_id, user_id, message) VALUES('$fileID', 'submitted', '$corrID', '$user', '{}');")
+    resp=$(psql -U ingest -h "$host" -d sda -At -c "INSERT INTO sda.file_event_log(file_id, event, user_id, message) VALUES('$fileID', 'submitted', '$user', '{}');")
     if [ "$(echo "$resp" | tr -d '\n')" != "INSERT 0 1" ]; then
         echo "insert file failed"
         exit 1
@@ -30,11 +30,23 @@ for host in migrate postgres; do
     archive_path=d853c51b-6aed-4243-b427-177f5e588857
     size="2035150"
     checksum="f03775a50feea74c579d459fdbeb27adafd543b87f6692703543a6ebe7daa1ff"
-    resp=$(psql -U ingest -h "$host" -d sda -At -c "SELECT sda.set_archived('$fileID', '$corrID', '$archive_path', '$size', '$checksum', 'SHA256');")
-    if [ "$resp" != "" ]; then
-        echo "mark file archived failed"
+
+    resp=$(psql -U ingest -h "$host" -d sda -At -c "UPDATE sda.files SET archive_file_path = '$archive_path', archive_file_size = '$size' WHERE id = '$fileID';")
+    if [ "$resp" != "UPDATE 1" ]; then
+        echo "update of files.archive_file_path, archive_file_size failed: $resp"
+        exit 1
+    fi
+    resp=$(psql -U ingest -h "$host" -d sda -At -c "INSERT INTO sda.checksums(file_id, checksum, type, source) VALUES('$fileID', '$checksum', upper('SHA256')::sda.checksum_algorithm, upper('UPLOADED')::sda.checksum_source);")
+    if [ "$(echo "$resp" | tr -d '\n')" != "INSERT 0 1" ]; then
+        echo "insert of archived checksum failed: $resp"
         exit 1
     fi
+    resp=$(psql -U ingest -h "$host" -d sda -At -c "INSERT INTO sda.file_event_log(file_id, event) VALUES('$fileID', 'archived');")
+    if [ "$(echo "$resp" | tr -d '\n')" != "INSERT 0 1" ]; then
+        echo "insert of file_event_log failed: $resp"
+        exit 1
+    fi
+
 done
 
 echo "30_ingest_queries completed successfully"

.github/integration/tests/postgres/40_verify_queries.sh

@@ -2,35 +2,55 @@
 set -eou pipefail
 
 export PGPASSWORD=verify
-corrID="33d29907-c565-4a90-98b4-e31b992ab376"
+fileID="33d29907-c565-4a90-98b4-e31b992ab376"
 
 for host in migrate postgres; do
-    fileID=$(psql -U verify -h "$host" -d sda -At -c "SELECT DISTINCT file_id from sda.file_event_log WHERE correlation_id = '$corrID';")
 
     ## get file status
-    status=$(psql -U verify -h "$host" -d sda -At -c "SELECT event from sda.file_event_log WHERE correlation_id = '$corrID' ORDER BY id DESC LIMIT 1;")
+    status=$(psql -U verify -h "$host" -d sda -At -c "SELECT event from sda.file_event_log WHERE file_id = '$fileID' ORDER BY id DESC LIMIT 1;")
     if [ "$status" = "" ]; then
-        echo "get file status failed"
+        echo "get file status failed: $resp"
         exit 1
     fi
 
     ## get file header
     header="637279707434676801000000010000006c00000000000000"
     dbheader=$(psql -U verify -h "$host" -d sda -At -c "SELECT header from sda.files WHERE id = '$fileID';")
     if [ "$dbheader" != "$header" ]; then
-        echo "wrong header received"
+        echo "wrong header received: $resp"
         exit 1
     fi
 
     ## mark file as 'COMPLETED'
     archive_checksum="64e56b0d245b819c116b5f1ad296632019490b57eeaebb419a5317e24a153852"
     decrypted_size="2034254"
     decrypted_checksum="febee6829a05772eea93c647e38bf5cc5bf33d1bcd0ea7d7bdd03225d84d2553"
-    resp=$(psql -U verify -h "$host" -d sda -At -c "SELECT sda.set_verified('$fileID', '$corrID', '$archive_checksum', 'SHA256', '$decrypted_size', '$decrypted_checksum', 'SHA256')")
-    if [ "$resp" != "" ]; then
-        echo "set_verified failed"
+
+    resp=$(psql -U verify -h "$host" -d sda -At -c "UPDATE sda.files SET decrypted_file_size = '$decrypted_size' WHERE id = '$fileID';")
+    if [ "$resp" != "UPDATE 1" ]; then
+        echo "update of files.decrypted_file_size failed: $resp"
+        exit 1
+    fi
+
+    resp=$(psql -U verify -h "$host" -d sda -At -c "INSERT INTO sda.checksums(file_id, checksum, type, source) VALUES('$fileID', '$archive_checksum', upper('SHA256')::sda.checksum_algorithm, upper('ARCHIVED')::sda.checksum_source);")
+    if [ "$(echo "$resp" | tr -d '\n')" != "INSERT 0 1" ]; then
+        echo "insert of archived checksum failed: $resp"
+        exit 1
+    fi
+
+    resp=$(psql -U verify -h "$host" -d sda -At -c "INSERT INTO sda.checksums(file_id, checksum, type, source) VALUES('$fileID', '$decrypted_checksum', upper('SHA256')::sda.checksum_algorithm, upper('UNENCRYPTED')::sda.checksum_source);")
+    if [ "$(echo "$resp" | tr -d '\n')" != "INSERT 0 1" ]; then
+        echo "insert of decrypted checksum failed: $resp"
+        exit 1
+    fi
+
+    resp=$(psql -U verify -h "$host" -d sda -At -c "INSERT INTO sda.file_event_log(file_id, event) VALUES('$fileID', 'verified');")
+    if [ "$(echo "$resp" | tr -d '\n')" != "INSERT 0 1" ]; then
+        echo "insert of file_event_log failed: $resp"
         exit 1
     fi
+
+
 done
 
 echo "40_verify_queries completed successfully"

.github/integration/tests/sda/21_cancel_test.sh

@@ -55,7 +55,7 @@ curl -k -u guest:guest "http://rabbitmq:15672/api/exchanges/sda/sda/publish" \
     -d "$cancel_body" | jq
 
 # check database to verify file status
-if [ "$(psql -U postgres -h postgres -d sda -At -c "select event from sda.file_event_log where correlation_id = '$CORRID' order by id DESC LIMIT 1")" != "disabled" ]; then
+if [ "$(psql -U postgres -h postgres -d sda -At -c "select event from sda.file_event_log where file_id = '$CORRID' order by id DESC LIMIT 1")" != "disabled" ]; then
     echo "canceling file failed"
     exit 1
 fi

.github/integration/tests/sda/22_error_test.sh

@@ -71,7 +71,7 @@ curl -k -u guest:guest "$URI/api/exchanges/sda/sda/publish" \
     -d "$ingest_body" | jq
 
 # check database to verify file status
-until [ "$(psql -U postgres -h postgres -d sda -At -c "SELECT event FROM sda.file_event_log WHERE correlation_id = '$CORRID' ORDER BY ID DESC LIMIT 1;")" = "error" ]; do
+until [ "$(psql -U postgres -h postgres -d sda -At -c "SELECT event FROM sda.file_event_log WHERE file_id = '$CORRID' ORDER BY ID DESC LIMIT 1;")" = "error" ]; do
     echo "waiting for file error to be logged by ingest"
     RETRY_TIMES=$((RETRY_TIMES + 1))
     if [ "$RETRY_TIMES" -eq 30 ]; then
@@ -83,7 +83,7 @@ done
 
 ## give the file a non existing archive path
 psql -U postgres -h postgres -d sda -Atq -c "UPDATE sda.files SET archive_file_path = '$CORRID', header = '637279707434676801000000010000006c00000000000000' WHERE id = '$CORRID';"
-psql -U postgres -h postgres -d sda -Atq -c "INSERT INTO sda.file_event_log(file_id, correlation_id, event) VALUES('$CORRID', '$CORRID', 'archived');"
+psql -U postgres -h postgres -d sda -Atq -c "INSERT INTO sda.file_event_log(file_id, event) VALUES('$CORRID', 'archived');"
 
 encrypted_checksums=$(
     jq -c -n \
@@ -119,7 +119,7 @@ curl -k -u guest:guest "$URI/api/exchanges/sda/sda/publish" \
 
 # check database to verify file status
 RETRY_TIMES=0
-until [ "$(psql -U postgres -h postgres -d sda -At -c "SELECT event FROM sda.file_event_log WHERE correlation_id = '$CORRID' ORDER BY ID DESC LIMIT 1;")" = "error" ]; do
+until [ "$(psql -U postgres -h postgres -d sda -At -c "SELECT event FROM sda.file_event_log WHERE file_id = '$CORRID' ORDER BY ID DESC LIMIT 1;")" = "error" ]; do
     echo "waiting for file error to be logged by verify"
     date
     RETRY_TIMES=$((RETRY_TIMES + 1))

.github/integration/tests/sda/31_cancel_test2.sh

@@ -7,13 +7,13 @@ ENC_SHA=$(sha256sum NA12878.bam.c4gh | cut -d' ' -f 1)
 ENC_MD5=$(md5sum NA12878.bam.c4gh | cut -d' ' -f 1)
 
 ## get correlation id from message
-CORRID=$(psql -U postgres -h postgres -d sda -At -c "select id from sda.files where submission_file_path = 'NA12878.bam.c4gh';")
+FILEID=$(psql -U postgres -h postgres -d sda -At -c "select id from sda.files where submission_file_path = 'NA12878.bam.c4gh';")
 
 
 properties=$(
     jq -c -n \
         --argjson delivery_mode 2 \
-        --arg correlation_id "$CORRID" \
+        --arg correlation_id "$FILEID" \
         --arg content_encoding UTF-8 \
         --arg content_type application/json \
         '$ARGS.named'
@@ -52,7 +52,7 @@ curl -k -u guest:guest "http://rabbitmq:15672/api/exchanges/sda/sda/publish" \
 
 # check database to verify file status
 RETRY_TIMES=0
-until [ "$(psql -U postgres -h postgres -d sda -At -c "select event from sda.file_event_log where correlation_id = '$CORRID' order by id DESC LIMIT 1;")" = "disabled" ]; do
+until [ "$(psql -U postgres -h postgres -d sda -At -c "select event from sda.file_event_log where file_id = '$FILEID' order by id DESC LIMIT 1;")" = "disabled" ]; do
     echo "canceling file failed"
     RETRY_TIMES=$((RETRY_TIMES + 1))
     if [ "$RETRY_TIMES" -eq 30 ]; then
@@ -132,7 +132,7 @@ curl -s -u guest:guest "http://rabbitmq:15672/api/exchanges/sda/sda/publish" \
     -d "$accession_body" | jq
 
 RETRY_TIMES=0
-until [ "$(psql -U postgres -h postgres -d sda -At -c "select event from sda.file_event_log where correlation_id = '$CORRID' order by id DESC LIMIT 1")" = "ready" ]; do
+until [ "$(psql -U postgres -h postgres -d sda -At -c "select event from sda.file_event_log where file_id = '$FILEID' order by id DESC LIMIT 1")" = "ready" ]; do
     echo "waiting for re-ingested file to become ready"
     RETRY_TIMES=$((RETRY_TIMES + 1))
     if [ "$RETRY_TIMES" -eq 30 ]; then

.github/integration/tests/sda/45_sync_test.sh

@@ -11,7 +11,7 @@ fi
 # check bucket for synced files
 for file in NA12878.bai NA12878_20k_b37.bai; do
     RETRY_TIMES=0
-    until [ "$(s3cmd -c direct ls s3://sync/test_dummy.org/"$file")" != "" ]; do
+    until [ "$(s3cmd -c direct ls s3://sync/"$file")" != "" ]; do
         RETRY_TIMES=$((RETRY_TIMES + 1))
         if [ "$RETRY_TIMES" -eq 30 ]; then
             echo "::error::Time out while waiting for files to be synced"