зеркало из https://github.com/microsoft/CCF.git
Use Unix domain socket (#4858)
Co-authored-by: Takuro Sato <takurosato@microsoft.com--username>
This commit is contained in:
Takuro Sato
GitHub
Родитель
0ca3aee7c1
Коммит
b23beabf37
|
|
@ -1,2 +1,2 @@
|
|||
<>??<><><><>
|
||||
<>
|
||||
<><>
|
||||
|
|
|
|||
|
|
@ -23,7 +23,7 @@ jobs:
|
|||
- script: |
|
||||
set -ex
|
||||
cd attestation-container
|
||||
docker build -t attestation-container .
|
||||
docker build -t attestation-container --build-arg build_test=True .
|
||||
az acr login --name attestationcontainerregistry
|
||||
login_server=$(az acr show --name attestationcontainerregistry --query loginServer --output tsv)
|
||||
image_version="ci-$(echo $(Build.BuildNumber) | sed 's/\.//g')"
|
||||
|
|
@ -38,7 +38,7 @@ jobs:
|
|||
python3.8 -m venv ./scripts/azure_deployment/.env
|
||||
source ./scripts/azure_deployment/.env/bin/activate
|
||||
pip install -r ./scripts/azure_deployment/requirements.txt
|
||||
python3.8 scripts/azure_deployment/arm_template.py deploy aci --subscription-id $(CCF_AZURE_SUBSCRIPTION_ID) --resource-group attestationContainer --aci-type dynamic-agent --deployment-name ci-$(Build.BuildNumber) --attestation-container-e2e --aci-file-share-name acshare --aci-file-share-account-name attestcontainercistorage --aci-storage-account-key $(ATTESTATION_CONTAINER_AZURE_STORAGE_KEY) > ~/aci_ips
|
||||
python3.8 scripts/azure_deployment/arm_template.py deploy aci --subscription-id $(CCF_AZURE_SUBSCRIPTION_ID) --resource-group attestationContainer --aci-type dynamic-agent --deployment-name ci-$(Build.BuildNumber) --attestation-container-e2e --ports 22 2522 --aci-file-share-name acshare --aci-file-share-account-name attestcontainercistorage --aci-storage-account-key $(ATTESTATION_CONTAINER_AZURE_STORAGE_KEY) > ~/aci_ips
|
||||
# Escape newlines: https://learn.microsoft.com/en-us/azure/devops/pipelines/process/set-variables-scripts?view=azure-devops&tabs=bash
|
||||
escape_data() {
|
||||
local data=$1
|
||||
|
|
@ -76,18 +76,6 @@ jobs:
|
|||
parameters:
|
||||
ssh_key: $(sshKey)
|
||||
|
||||
- script: |
|
||||
set -ex
|
||||
BUILD_NUMBER=$(echo $(Build.BuildNumber) | sed 's/\.//g')
|
||||
IP_ADDR=$(echo "$(IpAddresses)" | awk -v container_group="ci-$BUILD_NUMBER-business-logic-0" '$1 == container_group {print $2}')
|
||||
ssh agent@$IP_ADDR -o "StrictHostKeyChecking no" '
|
||||
workspace="/acci/workspace_$(Build.BuildNumber)"
|
||||
sudo chmod 600 -R $workspace
|
||||
sudo rm -rf $workspace
|
||||
'
|
||||
name: cleanup_workspace
|
||||
displayName: "Cleanup Workspace"
|
||||
|
||||
- script: |
|
||||
set -ex
|
||||
python3.8 -m venv ./scripts/azure_deployment/.env
|
||||
|
|
|
|||
|
|
@ -43,32 +43,27 @@ jobs:
|
|||
echo -e "Testing connection with $IP_ADDR"
|
||||
ssh agent@$IP_ADDR -o "StrictHostKeyChecking no" '
|
||||
echo "Connected successfully for port 22"
|
||||
workspace="/acci/workspace_$(Build.BuildNumber)"
|
||||
export GOPATH="$workspace/go" && mkdir -p $GOPATH
|
||||
export GOTMPDIR="$workspace/tmp" && mkdir -p $GOTMPDIR
|
||||
cd /usr/src/app/attest
|
||||
/usr/local/go/bin/go test
|
||||
# /usr/local/go/bin/go test
|
||||
./attest.test -test.v
|
||||
'
|
||||
name: run_unit_test
|
||||
displayName: "Unit Test in Attestation Container Instance Deployed ACIs"
|
||||
|
||||
- task: GoTool@0
|
||||
inputs:
|
||||
version: "1.19.3"
|
||||
|
||||
- template: azure_cli.yml
|
||||
parameters:
|
||||
app_id: $(ATTESTATION_CONTAINER_CI_APP_ID)
|
||||
service_principal_password: $(ATTESTATION_CONTAINER_CI_SERVICE_PRINCIPAL_PASSWORD)
|
||||
tenant: $(ATTESTATION_CONTAINER_CI_TENANT)
|
||||
displayName: "Unit Test in Attestation Container Instance Deployed to ACIs"
|
||||
|
||||
- script: |
|
||||
set -ex
|
||||
az account set --subscription $(CCF_AZURE_SUBSCRIPTION_ID)
|
||||
echo "Testing attestation container..."
|
||||
cd attestation-container
|
||||
echo "testing against ${IP_ADDR} ..."
|
||||
go test . --addr $IP_ADDR:50051 -v
|
||||
az container logs --resource-group attestationContainer --name ci-$(BUILD_NUMBER)-business-logic-0 --container-name ci-$(BUILD_NUMBER)-attestation-container
|
||||
echo -e "Running E2E test with $IP_ADDR"
|
||||
ssh -p 2522 agent@$IP_ADDR -o "StrictHostKeyChecking no" '
|
||||
echo "Connected successfully for port 2522"
|
||||
# workspace="/acci/workspace_$(Build.BuildNumber)/dummy-business-logic-container"
|
||||
# export GOPATH="$workspace/go" && mkdir -p $GOPATH
|
||||
# export GOTMPDIR="$workspace/tmp" && mkdir -p $GOTMPDIR
|
||||
cd /usr/src/app
|
||||
# /usr/local/go/bin/go test -v
|
||||
deployment_name="ci-$(echo $(Build.BuildNumber) | sed 's/\.//g')"
|
||||
echo $deployment_name
|
||||
socket_addr="/mnt/uds/sock"
|
||||
sudo ./attestation-container.test -addr $socket_addr -test.v
|
||||
'
|
||||
name: test_attestation_container
|
||||
displayName: "Test attestation container"
|
||||
displayName: "Test attestation container with dummy business logic container Instance Deployed to ACIs"
|
||||
|
|
|
|||
|
|
@ -1 +1,3 @@
|
|||
attestation-container
|
||||
attestation-container.test
|
||||
attest/attest.test
|
||||
|
|
|
|||
|
|
@ -2,9 +2,10 @@
|
|||
|
||||
# To run this:
|
||||
# docker build -t attestation-container .
|
||||
# docker run -it --rm -p 50051:50051 attestation-container
|
||||
# docker run -it --rm attestation-container
|
||||
|
||||
FROM golang:1.19
|
||||
ARG build_test=False
|
||||
|
||||
WORKDIR /usr/src/app
|
||||
|
||||
|
|
@ -13,6 +14,10 @@ COPY go.mod go.sum ./
|
|||
RUN go mod download && go mod verify
|
||||
|
||||
COPY . .
|
||||
|
||||
# Pre-compile tests to avoid I/O errors in ACI
|
||||
RUN if [ "$build_test" = "True" ] ; then go test -c && cd attest && go test -c && cd .. ; fi
|
||||
|
||||
RUN go build -v -o /usr/local/bin/app .
|
||||
|
||||
CMD ["app"]
|
||||
|
|
|
|||
|
|
@ -8,6 +8,7 @@ import (
|
|||
"log"
|
||||
"net"
|
||||
"os"
|
||||
"path/filepath"
|
||||
|
||||
"microsoft/attestation-container/attest"
|
||||
pb "microsoft/attestation-container/protobuf"
|
||||
|
|
@ -18,7 +19,7 @@ import (
|
|||
)
|
||||
|
||||
var (
|
||||
port = flag.Int("port", 50051, "The server port")
|
||||
socketAddress = flag.String("socket-address", "/tmp/attestation-container.sock", "The socket address of Unix domain socket (UDS)")
|
||||
endorsementServer = flag.String("endorsement-server", "Azure", "Server to fetch attestation endorsement. Value is either 'Azure' or 'AMD'")
|
||||
)
|
||||
|
||||
|
|
@ -57,7 +58,7 @@ func main() {
|
|||
fmt.Println("Attestation container started.")
|
||||
|
||||
if _, err := os.Stat(attest.SNP_DEVICE_PATH); err == nil {
|
||||
fmt.Printf("%s is detected", attest.SNP_DEVICE_PATH)
|
||||
fmt.Printf("%s is detected\n", attest.SNP_DEVICE_PATH)
|
||||
} else if errors.Is(err, os.ErrNotExist) {
|
||||
log.Fatalf("%s is not detected", attest.SNP_DEVICE_PATH)
|
||||
} else {
|
||||
|
|
@ -67,7 +68,21 @@ func main() {
|
|||
flag.Parse()
|
||||
validateFlags()
|
||||
|
||||
lis, err := net.Listen("tcp", fmt.Sprintf(":%d", *port))
|
||||
// Cleanup
|
||||
if _, err := os.Stat(*socketAddress); err == nil {
|
||||
if err := os.RemoveAll(*socketAddress); err != nil {
|
||||
log.Fatalf("Failed to clean up socket: %s", err)
|
||||
}
|
||||
}
|
||||
|
||||
// Create parent directory for socketAddress
|
||||
socketDir := filepath.Dir(*socketAddress)
|
||||
// os.MkdirAll doesn't return error when the directory already exists
|
||||
if err := os.MkdirAll(socketDir, os.ModePerm); err != nil {
|
||||
log.Fatalf("Failed to create directory for Unix domain socket: %s", err)
|
||||
}
|
||||
|
||||
lis, err := net.Listen("unix", *socketAddress)
|
||||
if err != nil {
|
||||
log.Fatalf("failed to listen: %v", err)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -5,17 +5,17 @@ import (
|
|||
"encoding/hex"
|
||||
"flag"
|
||||
"log"
|
||||
"net"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
pb "microsoft/attestation-container/protobuf"
|
||||
|
||||
"google.golang.org/grpc"
|
||||
"google.golang.org/grpc/credentials/insecure"
|
||||
)
|
||||
|
||||
var (
|
||||
addr = flag.String("addr", "localhost:50051", "the address to connect to")
|
||||
addr = flag.String("addr", "/tmp/attestation-container.sock", "the Unix domain socket address to connect to")
|
||||
)
|
||||
|
||||
const TIMEOUT_IN_SEC = 10
|
||||
|
|
@ -23,7 +23,10 @@ const TIMEOUT_IN_SEC = 10
|
|||
func TestFetchReport(t *testing.T) {
|
||||
flag.Parse()
|
||||
// Set up a connection to the server.
|
||||
conn, err := grpc.Dial(*addr, grpc.WithTransportCredentials(insecure.NewCredentials()))
|
||||
dialer := func(addr string, t time.Duration) (net.Conn, error) {
|
||||
return net.Dial("unix", addr)
|
||||
}
|
||||
conn, err := grpc.Dial(*addr, grpc.WithInsecure(), grpc.WithDialer(dialer))
|
||||
if err != nil {
|
||||
log.Fatalf("did not connect: %v", err)
|
||||
}
|
||||
|
|
@ -46,7 +49,10 @@ func TestFetchReport(t *testing.T) {
|
|||
func TestInputError(t *testing.T) {
|
||||
flag.Parse()
|
||||
// Set up a connection to the server.
|
||||
conn, err := grpc.Dial(*addr, grpc.WithTransportCredentials(insecure.NewCredentials()))
|
||||
dialer := func(addr string, t time.Duration) (net.Conn, error) {
|
||||
return net.Dial("unix", addr)
|
||||
}
|
||||
conn, err := grpc.Dial(*addr, grpc.WithInsecure(), grpc.WithDialer(dialer))
|
||||
if err != nil {
|
||||
log.Fatalf("did not connect: %v", err)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,6 +1,7 @@
|
|||
# Copyright (c) Microsoft Corporation. All rights reserved.
|
||||
# Licensed under the Apache 2.0 License.
|
||||
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import time
|
||||
|
|
@ -30,11 +31,12 @@ def get_pubkey():
|
|||
|
||||
|
||||
STARTUP_COMMANDS = {
|
||||
"dynamic-agent": lambda args: [
|
||||
"dynamic-agent": lambda args, ssh_port=22: [
|
||||
"apt-get update",
|
||||
"apt-get install -y openssh-server rsync sudo",
|
||||
"sed -i 's/PubkeyAuthentication no/PubkeyAuthentication yes/g' /etc/ssh/sshd_config",
|
||||
"sed -i 's/PasswordAuthentication yes/PasswordAuthentication no/g' /etc/ssh/sshd_config",
|
||||
f"sed -i 's/#\s*Port 22/Port {ssh_port}/g' /etc/ssh/sshd_config",
|
||||
"useradd -m agent",
|
||||
'echo "agent ALL=(ALL) NOPASSWD: ALL" >> /etc/sudoers',
|
||||
"service ssh restart",
|
||||
|
|
@ -81,10 +83,6 @@ scratch_mount := {"allowed": true}
|
|||
scratch_unmount := {"allowed": true}
|
||||
"""
|
||||
|
||||
# NOTE: It currently exposes the attestation-container's port (50051) to outside of the container group for e2e testing purpose,
|
||||
# but it shouldn't be done in actual CCF application.
|
||||
ATTESTATION_CONTAINER_PORT = 50051
|
||||
|
||||
|
||||
def make_dev_container_command(args):
|
||||
return [
|
||||
|
|
@ -98,7 +96,22 @@ def make_attestation_container_command(args):
|
|||
return [
|
||||
"/bin/sh",
|
||||
"-c",
|
||||
" && ".join([*STARTUP_COMMANDS["dynamic-agent"](args), "app"]),
|
||||
" && ".join(
|
||||
[
|
||||
*STARTUP_COMMANDS["dynamic-agent"](args),
|
||||
f"app -socket-address /mnt/uds/sock",
|
||||
]
|
||||
),
|
||||
]
|
||||
|
||||
|
||||
def make_dummy_business_logic_container_command(args, ssh_port):
|
||||
return [
|
||||
"/bin/sh",
|
||||
"-c",
|
||||
" && ".join(
|
||||
[*STARTUP_COMMANDS["dynamic-agent"](args, ssh_port), "tail -f /dev/null"]
|
||||
),
|
||||
]
|
||||
|
||||
|
||||
|
|
@ -128,12 +141,32 @@ def make_attestation_container(name, image, command, ports, with_volume):
|
|||
"command": command,
|
||||
"ports": [{"protocol": "TCP", "port": p} for p in ports],
|
||||
"environmentVariables": [],
|
||||
"resources": {"requests": {"memoryInGB": 16, "cpu": 4}},
|
||||
"resources": {"requests": {"memoryInGB": 8, "cpu": 2}},
|
||||
},
|
||||
}
|
||||
if with_volume:
|
||||
t["properties"]["volumeMounts"] = [
|
||||
{"name": "ccfcivolume", "mountPath": "/acci"}
|
||||
{"name": "ccfcivolume", "mountPath": "/acci"},
|
||||
{"name": "udsemptydir", "mountPath": "/mnt/uds"},
|
||||
]
|
||||
return t
|
||||
|
||||
|
||||
def make_dummy_business_logic_container(name, image, command, ports, with_volume):
|
||||
t = {
|
||||
"name": name,
|
||||
"properties": {
|
||||
"image": image,
|
||||
"command": command,
|
||||
"ports": [{"protocol": "TCP", "port": p} for p in ports],
|
||||
"environmentVariables": [],
|
||||
"resources": {"requests": {"memoryInGB": 8, "cpu": 2}},
|
||||
},
|
||||
}
|
||||
if with_volume:
|
||||
t["properties"]["volumeMounts"] = [
|
||||
{"name": "ccfcivolume", "mountPath": "/acci"},
|
||||
{"name": "udsemptydir", "mountPath": "/mnt/uds"},
|
||||
]
|
||||
return t
|
||||
|
||||
|
|
@ -174,7 +207,9 @@ def make_aci_deployment(parser: ArgumentParser) -> Deployment:
|
|||
parser.add_argument(
|
||||
"--ports",
|
||||
help="List of TCP ports to expose publicly on each container",
|
||||
action="append",
|
||||
action="extend",
|
||||
nargs="*",
|
||||
type=int,
|
||||
default=[22],
|
||||
)
|
||||
|
||||
|
|
@ -219,6 +254,11 @@ def make_aci_deployment(parser: ArgumentParser) -> Deployment:
|
|||
)
|
||||
|
||||
args = parser.parse_args()
|
||||
if len(args.ports) > 1:
|
||||
# Remove default value when ports are explicitly specified.
|
||||
# For example parser.parse_args() returns [22, 22, 2252] for '--ports 22 2252'.
|
||||
# This if block removes the first 22 because this behavior is not intuitive.
|
||||
args.ports = args.ports[1:]
|
||||
|
||||
# Note: Using ARM templates rather than Python SDK as ConfidentialComputeProperties does not work yet
|
||||
# with Python SDK (it should but isolationType cannot be specified - bug has been reported!)
|
||||
|
|
@ -244,16 +284,33 @@ def make_aci_deployment(parser: ArgumentParser) -> Deployment:
|
|||
)
|
||||
]
|
||||
else:
|
||||
# Attestation container E2E test requires two ports as `args.ports`: [<ssh for attestation container>, <ssh for dummy business logic container>]
|
||||
container_image = f"attestationcontainerregistry.azurecr.io/attestation-container:{args.deployment_name}"
|
||||
deployment_name = f"{args.deployment_name}-business-logic"
|
||||
container_name = f"{args.deployment_name}-attestation-container"
|
||||
command = make_attestation_container_command(args)
|
||||
args.ports.append(ATTESTATION_CONTAINER_PORT)
|
||||
container_name_dummy_blc = (
|
||||
f"{args.deployment_name}-dummy-business-logic-container"
|
||||
)
|
||||
command_dummy_blc = make_dummy_business_logic_container_command(
|
||||
args, args.ports[1]
|
||||
)
|
||||
with_volume = args.aci_file_share_name is not None
|
||||
containers = [
|
||||
make_attestation_container(
|
||||
container_name, container_image, command, args.ports, with_volume
|
||||
)
|
||||
container_name,
|
||||
container_image,
|
||||
command,
|
||||
args.ports[:1],
|
||||
with_volume,
|
||||
),
|
||||
make_dummy_business_logic_container(
|
||||
container_name_dummy_blc,
|
||||
container_image,
|
||||
command_dummy_blc,
|
||||
args.ports[1:],
|
||||
with_volume,
|
||||
),
|
||||
]
|
||||
|
||||
container_group_properties = {
|
||||
|
|
@ -277,7 +334,8 @@ def make_aci_deployment(parser: ArgumentParser) -> Deployment:
|
|||
"storageAccountName": args.aci_file_share_account_name,
|
||||
"storageAccountKey": args.aci_storage_account_key,
|
||||
},
|
||||
}
|
||||
},
|
||||
{"name": "udsemptydir", "emptyDir": {}},
|
||||
]
|
||||
|
||||
if not args.non_confidential:
|
||||
|
|
|
|||
Загрузка…
Ссылка в новой задаче