[Optionally] install Google Cloud Logging and Monitoring.

This has to be expicitly requested with arguments to InstallSpinnaker. The build_google_image script builds both these in by default, but an explicit install would not include these by default even on GCE.
2016-03-20 17:15:39 +00:00 · 2016-03-20 17:15:39 +00:00 · a7392f8578
--- a/InstallSpinnaker.sh
+++ b/InstallSpinnaker.sh
@ -53,6 +53,7 @@ function print_usage() {
 usage: $0 [--cloud_provider <aws|google|none|both>]
    [--aws_region <region>] [--google_region <region>]
    [--quiet] [--dependencies_only]
+    [--google_cloud_logging] [--google_cloud_monitoring]
    [--repository <debian repository url>]
    [--local-install] [--home_dir <path>]

@ -80,6 +81,16 @@ usage: $0 [--cloud_provider <aws|google|none|both>]
                                Only install the dependencies. This is intended
                                for development scenarios only

+    --google_cloud_logging      Install Google Cloud Logging support. This
+                                is independent of installing on Google Cloud
+                                Platform, but you may require additional
+                                authorization. See https://cloud.google.com/logging/docs/agent/authorization#install_private-key_authorization
+
+    --google_cloud_monitoring   Install Google Cloud Monitoring support. This
+                                is independent of installing on Google Cloud
+                                Platform, but you may require additional
+                                authorization. See https://cloud.google.com/monitoring/api/authentication
+
    --local-install             For Spinnaker and Java packages, download
                                packages and install using dpkg instead of
                                apt. Use this option only if you are having
@ -124,6 +135,12 @@ function process_args() {
          REPOSITORY_URL="$1"
          shift
          ;;
+      --google_cloud_logging)
+          GOOGLE_CLOUD_LOGGING="true"
+          ;;
+      --google_cloud_monitoring)
+          GOOGLE_CLOUD_MONITORING="true"
+          ;;
      --dependencies_only)
          CLOUD_PROVIDER="none"
          DEPENDENCIES_ONLY=true
@ -253,7 +270,7 @@ function set_aws_defaults_from_environ() {

 function set_defaults_from_environ() {
  local on_platform=""
-  local google_project_id=$(get_google_metadata_value "/project/project-id")
+  local google_project_id=$(get_google_metadata_value "project/project-id")

  if [[ -n "$google_project_id" ]]; then
      on_platform="google"
@ -280,7 +297,7 @@ function add_apt_repositories() {
  add-apt-repository -y ppa:chris-lea/redis-server
  # Cassandra
  # http://docs.datastax.com/en/cassandra/2.1/cassandra/install/installDeb_t.html
-  curl -L http://debian.datastax.com/debian/repo_key | apt-key add -
+  curl -s -L http://debian.datastax.com/debian/repo_key | apt-key add -
  echo "deb http://debian.datastax.com/community/ stable main" | tee /etc/apt/sources.list.d/datastax.list > /dev/null

  # Spinnaker
@ -290,7 +307,7 @@ function add_apt_repositories() {
    REPOSITORY_ORG=$(echo $REPOSITORY_URL | cut -d/ -f4)
    # Personal repositories might not be signed, so conditionally check.
    gpg=""
-    gpg=$(curl -f "https://bintray.com/user/downloadSubjectPublicKey?username=$REPOSITORY_ORG") || true
+    gpg=$(curl -s -f "https://bintray.com/user/downloadSubjectPublicKey?username=$REPOSITORY_ORG") || true
    if [ ! -z "$gpg" ]; then
        echo "$gpg" | apt-key add -
    fi   
@ -311,6 +328,43 @@ function install_java() {
  fi
 }

+function install_platform_dependencies() {
+  local google_scopes=$(get_google_metadata_value "instance/service-accounts/default/scopes")
+
+  if [[ -z "$google_scopes" ]]; then
+     # Not on GCP
+     if [[ "$GOOGLE_CLOUD_LOGGING" == "true" ]] \
+        || [[ "$GOOGLE_CLOUD_MONITORING" == "true" ]]; then
+          if [[ ! -f /etc/google/auth/application_default_credentials.json ]];
+          then
+            echo "You may need to add Google Project Credentials."
+            echo "See https://developers.google.com/identity/protocols/application-default-credentials"
+          fi
+     fi
+  fi
+
+  if [[ "$GOOGLE_CLOUD_LOGGING" == "true" ]]; then
+    # This can be installed on any platform, so dont scope to google.
+    # However, if on google, then certain scopes are required.
+    # The add_google_cloud_logging script checks the scope and warns.
+#    curl -s -L https://raw.githubusercontent.com/spinnaker/spinnaker/master/google/google_cloud_logging/add_google_cloud_logging.sh | sudo bash
+    curl -s -L https://raw.githubusercontent.com/ewiseblatt/spinnaker/install_platform_deps/google/google_cloud_logging/add_google_cloud_logging.sh | sudo bash
+  fi
+
+  if [[ "$GOOGLE_CLOUD_MONITORING" == "true" ]]; then
+    # This can be installed on any platform, so dont scope to google.
+    # However, if on google, then certain scopes are required.
+    curl -s https://repo.stackdriver.com/stack-install.sh | sudo bash
+    if [[ ! -z "$google_scopes" ]] && [[ $scopes != *"monitoring.write"* ]]; then
+      # This is not necessarily bad because we might be using this instance
+      # to create an image (e.g. packer). Only the runtime instances need
+      # this scope.
+      echo "Missing scope 'https://www.googleapis.com/auth/monitoring.write'"
+      echo "Google Cloud Monitoring will not be able to send data upstream."
+    fi
+  fi
+}
+
 function install_dependencies() {
  # java
  if [ "$DOWNLOAD" != "true" ];then
@ -506,13 +560,14 @@ fi

 install_java
 install_apache2
+install_platform_dependencies
 install_dependencies
 install_redis_server
 install_cassandra

 ## Packer
 mkdir /tmp/packer && pushd /tmp/packer
-curl -L -O https://releases.hashicorp.com/packer/0.8.6/packer_0.8.6_linux_amd64.zip
+curl -s -L -O https://releases.hashicorp.com/packer/0.8.6/packer_0.8.6_linux_amd64.zip
 unzip -u -o -q packer_0.8.6_linux_amd64.zip -d /usr/bin
 popd
 rm -rf /tmp/packer
--- a/dev/build_google_image.packer
+++ b/dev/build_google_image.packer
@ -8,7 +8,9 @@
    "zone": "us-central1-f",
    "source_image": null,
    "base_srcdir": null,
-    "target_image": "{{env `USER`}}-spinnaker-{{timestamp}}"
+    "target_image": "{{env `USER`}}-spinnaker-{{timestamp}}",
+    "google_cloud_logging": "true",
+    "google_cloud_monitoring": "true"
 },

 "builders": [{
@ -26,39 +28,14 @@
    "source": "{{user `install_path`}}",
    "destination": "/tmp/install_spinnaker.sh"
  },
-  {
-    "type": "file",
-    "source": "{{user `base_srcdir`}}/google/google_cloud_logging/spinnaker.conf",
-    "destination": "/tmp/spinnaker_cloud_logging.conf"
-  },
-  {
-    "type": "shell",
-    "inline": [
-      "cd /tmp",
-      "mkdir -p /etc/google-fluentd/config.d",
-      "mv spinnaker_cloud_logging.conf /etc/google-fluentd/config.d/spinnaker.conf",
-      "curl -s -O https://dl.google.com/cloudagents/install-logging-agent.sh",
-      "bash ./install-logging-agent.sh",
-      "rm ./install-logging-agent.sh",
-      "service google-fluentd stop"
-    ]
-  },
-  {
-    "type": "shell",
-    "inline": [
-      "cd /tmp",
-      "CODENAME=$(lsb_release -sc)",
-      "curl -s -S -f -o /etc/apt/sources.list.d/stackdriver.list https://repo.stackdriver.com/$CODENAME.list",
-      "curl -s -f https://app.stackdriver.com/RPM-GPG-KEY-stackdriver | apt-key add -",
-      "apt-get -q update",
-      "DEBIAN_FRONTEND=noninteractive apt-get -y -q install stackdriver-agent"
-    ]
-  },
  {
    "type": "shell",
    "inline": [
+        "flags=\"--cloud_provider google --google_region us-central1 --quiet\"",
+        "if \"{{ user `google_cloud_logging` }}\" == \"true\"; then flags=\"$flags --google_cloud_logging\"; fi",
+        "if \"{{ user `google_cloud_monitoring` }}\" == \"true\"; then flags=\"$flags --google_cloud_monitoring\"; fi",
        "chmod +x /tmp/install_spinnaker.sh",
-        "/tmp/install_spinnaker.sh --repository {{user `debian_repo`}} --cloud_provider google --google_region us-central1 --quiet",
+        "/tmp/install_spinnaker.sh --repository {{user `debian_repo`}} $flags",
        "service spinnaker stop"
     ]
  },
--- a/dev/build_google_image.sh
+++ b/dev/build_google_image.sh
@ -14,7 +14,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.

-CURRENT_IMAGE=$(gcloud compute images list 2>1 | grep ubuntu-1404 | tail -1 | awk '{print $1}')
+CURRENT_IMAGE=$(gcloud compute images list 2>&1 | grep ubuntu-1404 | tail -1 | awk '{print $1}')

 VARS="-var 'install_path=$(dirname $0)/../InstallSpinnaker.sh'"
 VARS="$VARS -var 'source_image=$CURRENT_IMAGE'"
--- a/google/google_cloud_logging/add_google_cloud_logging.sh
+++ b/google/google_cloud_logging/add_google_cloud_logging.sh
@ -22,15 +22,17 @@ GOOGLE_METADATA_URL="http://metadata.google.internal/computeMetadata/v1"

 scopes=$(curl -s -L -H "Metadata-Flavor:Google" "$GOOGLE_METADATA_URL/instance/service-accounts/default/scopes")
 if [[ $? -eq 0 ]] && [[ $scopes != *"logging.write"* ]]; then
-  echo "ERROR - missing scope 'https://www.googleapis.com/auth/logging.write'"
+  echo "WARNING - missing scope 'https://www.googleapis.com/auth/logging.write'"
  echo "---------------------------------------------------------------------"
  echo "Have scopes:"
  echo "$scopes"
  echo ""
-  echo "You will need to create a new VM that adds the scope for"
-  echo "   https://www.googleapis.com/auth/logging.write"
-  echo "then try this script again (in the new instance!)."
-  exit -1
+  echo "You will not be able to upload logging data to Google Cloud Logging"
+  echo "unless you add https://www.googleapis.com/auth/logging.write"
+  echo "as an Authorization Scope when creating your VM."
+  echo ""
+  # Proceed anyway. Note that packer will not have this scope, but that is
+  # ok since only the instance from the image packer creates needs it.
 fi

 sudo mkdir -p /etc/google-fluentd/config.d
@ -42,7 +44,7 @@ else
  # Otherwise, download the config file from github.
  SPINNAKER_CONF_URL="https://raw.githubusercontent.com/spinnaker/spinnaker/master/google/google_cloud_logging/spinnaker.conf"
  cd /etc/google-fluentd/config.d
-  sudo sudo curl -s -O "$SPINNAKER_CONF_URL"
+  sudo curl -s -O "$SPINNAKER_CONF_URL"
 fi

 cd /tmp