Fix build failure, update package versions (#41)

* gdal version needs to be locked in pyproject.toml to the same version in the Dockerfile. Otherwise when new versions of gdal come out, the build will attempt to use them and that may fail since the native code won't be installed and pip will try to build the wheel and fail. * Update stac-browser to 3.0.2 which fixes a build issue where upstream packages changed and previous versions (3.0.0, 3.0.1) will no longer build. * Update packages to latest versions. * Remove use of xargs and replace with a for loop to support cross-platform use of the script. * Fix retrieval of signed-in user object ID. If no value was found in the id field, then try objectId. If that is not found, then fail the script. * Pass cloudName to stac-scaler deployment so deployment works on sovereign or other non-public clouds. * Switch to stac-fastapi-pgstac, since the pgstac backend was broken out of the stac-fastapi project starting with version 2.4.6. * Update naip deployment docs to make copy/paste easier
2023-06-16 16:49:38 -04:00 · 2023-06-16 16:49:38 -04:00 · 105c1af9c0
--- a/deploy/helm/stac-scaler/values.yaml
+++ b/deploy/helm/stac-scaler/values.yaml
@ -106,7 +106,7 @@ stacfastapi:
  labels:
    app: app-stacfastapi
  image:
-    name: stac-fastapi
+    name: stac-fastapi-pgstac
    pullPolicy: Always
    tag: "latest"
  env:
--- a/deploy/scripts/build.sh
+++ b/deploy/scripts/build.sh
@ -10,8 +10,8 @@ ENV_CODE=${1:-${ENV_CODE}}

 set -ae
 ENV_NAME=${ENV_NAME:-"stac-${ENV_CODE}"}
-STAC_FASTAPI_VERSION=${STAC_FASTAPI_VERSION:-"2.4.5"}
-STAC_BROWSER_VERSION=${STAC_BROWSER_VERSION:-"3.0.0"}
+STAC_FASTAPI_VERSION=${STAC_FASTAPI_VERSION:-"2.4.8"}
+STAC_BROWSER_VERSION=${STAC_BROWSER_VERSION:-"3.0.2"}
 PROCESSING_RESOURCE_GROUP=${PROCESSING_RESOURCE_GROUP:-"${ENV_CODE}-processing-rg"}

 ACR_NAME=$(az acr list -g ${PROCESSING_RESOURCE_GROUP} \
@ -20,16 +20,16 @@ ACR_NAME=$(az acr list -g ${PROCESSING_RESOURCE_GROUP} \
 echo "Building stac-cli Docker image in ACR"
 az acr build -o none --no-logs --registry $ACR_NAME --image stac-cli ${PRJ_ROOT}

-# build stac-fastapi from https://github.com/stac-utils/stac-fastapi/archive/refs/tags/${STAC_FASTAPI_VERSION}.tar.gz
-STAC_FASTAPI_SRC_DIR=${STAC_FASTAPI_SRC_DIR:-"${PRJ_ROOT}/src/stac_fastapi_k8s/src"}
-STAC_FASTAPI_RELEASE_URI=${STAC_FASTAPI_RELEASE_URI:-"https://github.com/stac-utils/stac-fastapi/archive/refs/tags/${STAC_FASTAPI_VERSION}.tar.gz"}
+# build stac-fastapi-pgstac from https://github.com/stac-utils/stac-fastapi-pgstac/archive/refs/tags/${STAC_FASTAPI_VERSION}.tar.gz
+STAC_FASTAPI_SRC_DIR=${STAC_FASTAPI_SRC_DIR:-"${PRJ_ROOT}/src/stac_fastapi_pgstac_k8s/src"}
+STAC_FASTAPI_RELEASE_URI=${STAC_FASTAPI_RELEASE_URI:-"https://github.com/stac-utils/stac-fastapi-pgstac/archive/refs/tags/${STAC_FASTAPI_VERSION}.tar.gz"}
 mkdir -p $STAC_FASTAPI_SRC_DIR
 wget $STAC_FASTAPI_RELEASE_URI -P $STAC_FASTAPI_SRC_DIR
 tar xf ${STAC_FASTAPI_SRC_DIR}/${STAC_FASTAPI_VERSION}.tar.gz -C ${STAC_FASTAPI_SRC_DIR}
-echo "Building stac-fastapi Docker image in ACR"
-az acr build -o none --no-logs --registry $ACR_NAME --image stac-fastapi \
-  --file ${STAC_FASTAPI_SRC_DIR}/stac-fastapi-${STAC_FASTAPI_VERSION}/docker/Dockerfile \
-  ${STAC_FASTAPI_SRC_DIR}/stac-fastapi-${STAC_FASTAPI_VERSION}
+echo "Building stac-fastapi-pgstac Docker image in ACR"
+az acr build -o none --no-logs --registry $ACR_NAME --image stac-fastapi-pgstac \
+  --file ${STAC_FASTAPI_SRC_DIR}/stac-fastapi-pgstac-${STAC_FASTAPI_VERSION}/Dockerfile \
+  ${STAC_FASTAPI_SRC_DIR}/stac-fastapi-pgstac-${STAC_FASTAPI_VERSION}
 rm -rf ${STAC_FASTAPI_SRC_DIR}

 # build stac-browser from https://github.com/radiantearth/stac-browser/archive/refs/tags/v${STAC_BROWSER_VERSION}.tar.gz
--- a/deploy/scripts/configure.sh
+++ b/deploy/scripts/configure.sh
@ -225,6 +225,7 @@ helm upgrade --install stac-scaler ${PRJ_ROOT}/deploy/helm/stac-scaler \
    --set serviceAccountName=${AKS_SERVICE_ACCOUNT_NAME} \
    --set keyVaultName=${KEY_VAULT_NAME} \
    --set tenantId=${IDENTITY_TENANT} \
+    --set cloudName=$(az cloud show --query name -otsv) \
    --set processors.staccollection.env.DATA_STORAGE_ACCOUNT_NAME=${DATA_STORAGE_ACCOUNT_NAME} \
    --set processors.staccollection.env.STACCOLLECTION_STORAGE_CONTAINER_NAME=${STACCOLLECTION_STORAGE_CONTAINER_NAME} \
    --set processors.staccollection.env.PGHOST=${PGHOST} \
@ -487,7 +488,7 @@ spec:
          service:
            name: stac-browser
            port:
-              number: 8082
+              number: 8080
 EOF

 # Deploy the auth proxy if selected
--- a/deploy/scripts/install.sh
+++ b/deploy/scripts/install.sh
@ -38,13 +38,17 @@ ENABLE_PUBLIC_ACCESS=${8:-${ENABLE_PUBLIC_ACCESS:-"false"}}
 USER_OBJ_ID=${USER_OBJ_ID:-"$(az ad signed-in-user show --query id --output tsv 2> /dev/null || echo '')"}
 POSTGRES_PRIVATE_ENDPOINT_DISABLED=${POSTGRES_PRIVATE_ENDPOINT_DISABLED:-false}
 if [[ -n "$DNS_PREFIX" ]]; then
-  DNS_ARG="ingressPublicIPDnsPrefixVar=$DNS_PREFIX"
+  DNS_ARG="ingressPublicIPDnsPrefix=$DNS_PREFIX"
 fi

-if [[ -z "$USER_OBJ_ID" ]]
-  then
+if [[ -z "$USER_OBJ_ID" ]]; then
+  # If there was no "id" field, then try "objectId"
+  USER_OBJ_ID="$(az ad signed-in-user show --query objectId --output tsv 2> /dev/null || echo '')"
+fi
+if [[ -z "$USER_OBJ_ID" ]]; then
    echo "Set USER_OBJ_ID environment variable after retrieving the value from device with an approved MDM provider like Intune"
    echo "To get USER_OBJ_ID, run 'az ad signed-in-user show --query id --output tsv'"
+    exit 1
 fi

 # Register the workload identity feature, and wait for it to be registered.
@ -83,4 +87,6 @@ DEPLOYMENT_SCRIPT="az deployment sub create -o none -l $LOCATION -n $DEPLOYMENT_

 $DEPLOYMENT_SCRIPT

-az deployment-scripts list -g "${ENV_CODE}-vnet-rg" --query "[].name" -o tsv | xargs -otl az deployment-scripts delete -g "${ENV_CODE}-vnet-rg" $_ --yes --name
+for script in $(az deployment-scripts list -g "${ENV_CODE}-vnet-rg" --query "[].name" -o tsv); do
+  az deployment-scripts delete -g "${ENV_CODE}-vnet-rg" --yes --name $script
+done
--- a/docs/cataloging-sample-data.md
+++ b/docs/cataloging-sample-data.md
@ -16,8 +16,13 @@ A STAC Collection is created from a [json document](../deploy/sample-data/collec
 that is uploaded to the appropriate container in the Storage Account.

    ```bash
+    # Replace with the name of your storage account, or update the entire URL for sovereign clouds
+    export STORAGE_BLOB_ENDPOINT=https://<data-storage-account>.blob.core.windows.net/
+    # Generate a SAS token for your storage account
+    export SAS_TOKEN=<SAS-token-for-data-storage-account>
+    
    curl -s https://raw.githubusercontent.com/Azure/Azure-Orbital-STAC/main/deploy/sample-data/collection_naip_test.json | \
-    azcopy copy "https://<data-storage-account>.blob.core.windows.net/staccollection/collection_naip_test.json?<SAS-token-for-data-storage-account>" --from-to PipeBlob
+    azcopy copy "${STORAGE_BLOB_ENDPOINT}staccollection/collection_naip_test.json?${SAS_TOKEN}" --from-to PipeBlob
    ```

 2. Generate the SAS token for getting the data from Planetary Computers.
@ -52,13 +57,13 @@ Addition of the STAC Item to a STAC Collection is two step process.
    a. Upload metadata

    ```bash
-    azcopy copy "https://naipeuwest.blob.core.windows.net/naip/v002/wa/2015/wa_fgdc_2015/45117?${TOKEN}" "https://<data-storage-account>.blob.core.windows.net/stacify/v002/wa/2015/wa_fgdc_2015?<SAS-token-for-data-storage-account>" --recursive=true
+    azcopy copy "https://naipeuwest.blob.core.windows.net/naip/v002/wa/2015/wa_fgdc_2015/45117?${TOKEN}" "${STORAGE_BLOB_ENDPOINT}stacify/v002/wa/2015/wa_fgdc_2015?${SAS_TOKEN}" --recursive=true
    ```

    b. Upload raster data

    ```bash
-    azcopy copy "https://naipeuwest.blob.core.windows.net/naip/v002/wa/2015/wa_100cm_2015/45117?${TOKEN}" "https://<data-storage-account>.blob.core.windows.net/stacify/v002/wa/2015/wa_100cm_2015?<SAS-token-for-data-storage-account>" --recursive=true
+    azcopy copy "https://naipeuwest.blob.core.windows.net/naip/v002/wa/2015/wa_100cm_2015/45117?${TOKEN}" "${STORAGE_BLOB_ENDPOINT}stacify/v002/wa/2015/wa_100cm_2015?${SAS_TOKEN}" --recursive=true
    ```

 ## Steps to validate Cataloged data
@ -66,7 +71,8 @@ Addition of the STAC Item to a STAC Collection is two step process.
 Run the below curl command for validation:
  
 ```bash
-curl <Gateway URL>/api/collections/naip | json_pp
+export GATEWAY_URL=<Gateway URL>
+curl ${GATEWAY_URL}/api/collections/naip | json_pp

-curl <Gateway URL>/api/collections/naip/items | json_pp
+curl ${GATEWAY_URL}/api/collections/naip/items | json_pp
 ```
--- a/pyproject.toml
+++ b/pyproject.toml
@ -21,19 +21,19 @@ classifiers = [
 dependencies = [
  "knack >= 0.10.0",
  "opencensus-ext-azure >= 1.1.9",
-  "azure-servicebus == 7.8.3",
-  "azure-storage-blob == 12.15.0",
-  "pystac == 1.7.1",
+  "azure-servicebus == 7.11.0",
+  "azure-storage-blob == 12.16.0",
+  "pystac == 1.7.3",
  "python_dateutil == 2.8.2",
  "python_dotenv == 1.0.0",
-  "rasterio == 1.3.6",
-  "stactools == 0.4.5",
-  "psycopg[binary] == 3.1.8",
-  "pypgstac[psycopg] == 0.7.3",
-  # Don't set a version for GDAL, since it will likely be pre-installed
-  # on the system (e.g., on Linux) and we won't want to force an upgrade
-  # or downgrade.
-  "gdal",
+  "rasterio == 1.3.7",
+  "stactools == 0.4.8",
+  "psycopg[binary] == 3.1.9",
+  "pypgstac[psycopg] == 0.7.10",
+  # This version should match the version of GDAL that is pre-installled
+  # on the system (e.g., on Linux). See the base image version in Dockerfile
+  # for the version of GDAL that is pre-installed when building for Docker.
+  "gdal == 3.6.3",
 ]

 [project.urls]