Migrate to storage split library
- Remove queue deletion code - Resolves #133
This commit is contained in:
Fred Park
Родитель
7c87b043a0
Коммит
607bfd252e
|
|
@ -1,3 +1,3 @@
|
|||
azure-batch==4.0.0
|
||||
msrest==0.4.15
|
||||
msrest==0.4.16
|
||||
requests==2.18.4
|
||||
|
|
|
|||
|
|
@ -10,26 +10,23 @@ MAINTAINER Fred Park <https://github.com/Azure/batch-shipyard>
|
|||
COPY --from=0 /usr/lib/libtorrent-rasterbar.so.8.0.0 /usr/lib/
|
||||
COPY --from=0 /usr/lib/python3.6/site-packages/*libtorrent* /usr/lib/python3.6/site-packages/
|
||||
|
||||
# add dependencies, symlink libtorrent, create script directory
|
||||
# copy in files
|
||||
COPY cascade.py perf.py docker_cascade.sh requirements.txt /opt/batch-shipyard/
|
||||
|
||||
# add dependencies, symlink libtorrent, compile python files
|
||||
RUN apk update \
|
||||
&& apk add --update --no-cache \
|
||||
musl build-base python3 python3-dev libressl-dev libffi-dev \
|
||||
ca-certificates boost boost-python3 boost-random \
|
||||
tar pigz docker bash \
|
||||
&& pip3 install --no-cache-dir --upgrade azure-storage==0.36.0 \
|
||||
&& pip3 install --no-cache-dir --upgrade -r /opt/batch-shipyard/requirements.txt \
|
||||
&& ln -s /usr/lib/libboost_python3.so.1.62.0 /usr/lib/libboost_python.so.1.62.0 \
|
||||
&& ln -s /usr/lib/libtorrent-rasterbar.so.8.0.0 /usr/lib/libtorrent-rasterbar.so \
|
||||
&& ln -s /usr/lib/libtorrent-rasterbar.so.8.0.0 /usr/lib/libtorrent-rasterbar.so.8 \
|
||||
&& ldconfig /usr/lib \
|
||||
&& apk del --purge build-base python3-dev libressl-dev libffi-dev \
|
||||
&& rm /var/cache/apk/* \
|
||||
&& mkdir -p /opt/batch-shipyard
|
||||
|
||||
# copy in files
|
||||
COPY cascade.py perf.py docker_cascade.sh /opt/batch-shipyard/
|
||||
|
||||
# pre-compile files
|
||||
RUN python3 -m compileall -f /opt/batch-shipyard
|
||||
&& python3 -m compileall -f /opt/batch-shipyard
|
||||
|
||||
# set command
|
||||
CMD ["/opt/batch-shipyard/docker_cascade.sh"]
|
||||
|
|
|
|||
|
|
@ -42,8 +42,8 @@ import threading
|
|||
import time
|
||||
# non-stdlib imports
|
||||
import azure.common
|
||||
import azure.cosmosdb.table as azuretable
|
||||
import azure.storage.blob as azureblob
|
||||
import azure.storage.table as azuretable
|
||||
try:
|
||||
import libtorrent
|
||||
_LIBTORRENT_IMPORTED = True
|
||||
|
|
@ -286,11 +286,11 @@ def add_dht_node(ip: str, port: int):
|
|||
|
||||
def _renew_blob_lease(
|
||||
loop: asyncio.BaseEventLoop,
|
||||
blob_client: azure.storage.blob.BlockBlobService,
|
||||
blob_client: azureblob.BlockBlobService,
|
||||
container_key: str, resource: str, blob_name: str):
|
||||
"""Renew a storage blob lease
|
||||
:param asyncio.BaseEventLoop loop: event loop
|
||||
:param azure.storage.blob.BlockBlobService blob_client: blob client
|
||||
:param azureblob.BlockBlobService blob_client: blob client
|
||||
:param str container_key: blob container index into _STORAGE_CONTAINERS
|
||||
:param str resource: resource
|
||||
:param str blob_name: blob name
|
||||
|
|
@ -346,12 +346,12 @@ def compute_resource_hash(resource: str) -> str:
|
|||
class DockerSaveThread(threading.Thread):
|
||||
"""Docker Save Thread"""
|
||||
def __init__(
|
||||
self, blob_client: azure.storage.blob.BlockBlobService,
|
||||
table_client: azure.storage.table.TableService,
|
||||
self, blob_client: azureblob.BlockBlobService,
|
||||
table_client: azuretable.TableService,
|
||||
resource: str, blob_name: str, nglobalresources: int):
|
||||
"""DockerSaveThread ctor
|
||||
:param azure.storage.blob.BlockBlobService blob_client: blob client
|
||||
:param azure.storage.table.TableService table_client: table client
|
||||
:param azureblob.BlockBlobService blob_client: blob client
|
||||
:param azuretable.TableService table_client: table client
|
||||
:param str resource: resource
|
||||
:param str blob_name: resource blob name
|
||||
:param int nglobalresources: number of global resources
|
||||
|
|
@ -573,13 +573,13 @@ class DockerSaveThread(threading.Thread):
|
|||
|
||||
async def _direct_download_resources_async(
|
||||
loop: asyncio.BaseEventLoop,
|
||||
blob_client: azure.storage.blob.BlockBlobService,
|
||||
table_client: azure.storage.table.TableService,
|
||||
blob_client: azureblob.BlockBlobService,
|
||||
table_client: azuretable.TableService,
|
||||
ipaddress: str, nglobalresources: int) -> None:
|
||||
"""Direct download resource logic
|
||||
:param asyncio.BaseEventLoop loop: event loop
|
||||
:param azure.storage.blob.BlockBlobService blob_client: blob client
|
||||
:param azure.storage.table.TableService table_client: table client
|
||||
:param azureblob.BlockBlobService blob_client: blob client
|
||||
:param azuretable.TableService table_client: table client
|
||||
:param str ipaddress: ip address
|
||||
:param int nglobalresources: number of global resources
|
||||
"""
|
||||
|
|
@ -663,10 +663,10 @@ async def _direct_download_resources_async(
|
|||
|
||||
|
||||
def _merge_service(
|
||||
table_client: azure.storage.table.TableService,
|
||||
table_client: azuretable.TableService,
|
||||
resource: str, nglobalresources: int) -> None:
|
||||
"""Merge entity to services table
|
||||
:param azure.storage.table.TableService table_client: table client
|
||||
:param azuretable.TableService table_client: table client
|
||||
:param str resource: resource to add to services table
|
||||
:param int nglobalresources: number of global resources
|
||||
"""
|
||||
|
|
@ -761,12 +761,12 @@ def _log_torrent_info(resource: str, th) -> None:
|
|||
|
||||
def bootstrap_dht_nodes(
|
||||
loop: asyncio.BaseEventLoop,
|
||||
table_client: azure.storage.table.TableService,
|
||||
table_client: azuretable.TableService,
|
||||
ipaddress: str,
|
||||
num_attempts: int) -> None:
|
||||
"""Bootstrap DHT router nodes
|
||||
:param asyncio.BaseEventLoop loop: event loop
|
||||
:param azure.storage.table.TableService table_client: table client
|
||||
:param azuretable.TableService table_client: table client
|
||||
:param str ipaddress: ip address
|
||||
:param int num_attempts: number of attempts
|
||||
"""
|
||||
|
|
@ -859,11 +859,11 @@ class DockerLoadThread(threading.Thread):
|
|||
|
||||
async def _load_and_register_async(
|
||||
loop: asyncio.BaseEventLoop,
|
||||
table_client: azure.storage.table.TableService,
|
||||
table_client: azuretable.TableService,
|
||||
nglobalresources: int) -> None:
|
||||
"""Load and register image
|
||||
:param asyncio.BaseEventLoop loop: event loop
|
||||
:param azure.storage.table.TableService table_client: table client
|
||||
:param azuretable.TableService table_client: table client
|
||||
:param int nglobalresource: number of global resources
|
||||
"""
|
||||
global _LR_LOCK_ASYNC
|
||||
|
|
@ -892,11 +892,11 @@ async def _load_and_register_async(
|
|||
|
||||
async def manage_torrents_async(
|
||||
loop: asyncio.BaseEventLoop,
|
||||
table_client: azure.storage.table.TableService,
|
||||
table_client: azuretable.TableService,
|
||||
ipaddress: str, nglobalresources: int) -> None:
|
||||
"""Manage torrents
|
||||
:param asyncio.BaseEventLoop loop: event loop
|
||||
:param azure.storage.table.TableService table_client: table client
|
||||
:param azuretable.TableService table_client: table client
|
||||
:param str ipaddress: ip address
|
||||
:param int nglobalresource: number of global resources
|
||||
"""
|
||||
|
|
@ -943,13 +943,13 @@ async def manage_torrents_async(
|
|||
|
||||
async def download_monitor_async(
|
||||
loop: asyncio.BaseEventLoop,
|
||||
blob_client: azure.storage.blob.BlockBlobService,
|
||||
table_client: azure.storage.table.TableService,
|
||||
blob_client: azureblob.BlockBlobService,
|
||||
table_client: azuretable.TableService,
|
||||
ipaddress: str, nglobalresources: int) -> None:
|
||||
"""Download monitor
|
||||
:param asyncio.BaseEventLoop loop: event loop
|
||||
:param azure.storage.blob.BlockBlobService blob_client: blob client
|
||||
:param azure.storage.table.TableService table_client: table client
|
||||
:param azureblob.BlockBlobService blob_client: blob client
|
||||
:param azuretable.TableService table_client: table client
|
||||
:param str ipaddress: ip address
|
||||
:param int nglobalresource: number of global resources
|
||||
"""
|
||||
|
|
@ -977,10 +977,10 @@ async def download_monitor_async(
|
|||
|
||||
|
||||
def _get_torrent_num_seeds(
|
||||
table_client: azure.storage.table.TableService,
|
||||
table_client: azuretable.TableService,
|
||||
resource: str) -> int:
|
||||
"""Get number of torrent seeders via table
|
||||
:param azure.storage.table.TableService table_client: table client
|
||||
:param azuretable.TableService table_client: table client
|
||||
:param int nglobalresource: number of global resources
|
||||
:rtype: int
|
||||
:return: number of seeds
|
||||
|
|
@ -996,12 +996,12 @@ def _get_torrent_num_seeds(
|
|||
|
||||
|
||||
def _start_torrent_via_storage(
|
||||
blob_client: azure.storage.blob.BlockBlobService,
|
||||
table_client: azure.storage.table.TableService,
|
||||
blob_client: azureblob.BlockBlobService,
|
||||
table_client: azuretable.TableService,
|
||||
resource: str, entity: dict=None) -> None:
|
||||
"""Start a torrent via storage entity
|
||||
:param azure.storage.blob.BlockBlobService blob_client: blob client
|
||||
:param azure.storage.table.TableService table_client: table client
|
||||
:param azureblob.BlockBlobService blob_client: blob client
|
||||
:param azuretable.TableService table_client: table client
|
||||
:param str resource: resource
|
||||
:param dict entity: entity
|
||||
"""
|
||||
|
|
@ -1037,12 +1037,12 @@ def _start_torrent_via_storage(
|
|||
|
||||
|
||||
def _check_resource_has_torrent(
|
||||
blob_client: azure.storage.blob.BlockBlobService,
|
||||
table_client: azure.storage.table.TableService,
|
||||
blob_client: azureblob.BlockBlobService,
|
||||
table_client: azuretable.TableService,
|
||||
resource: str) -> bool:
|
||||
"""Check if a resource has an associated torrent
|
||||
:param azure.storage.blob.BlockBlobService blob_client: blob client
|
||||
:param azure.storage.table.TableService table_client: table client
|
||||
:param azureblob.BlockBlobService blob_client: blob client
|
||||
:param azuretable.TableService table_client: table client
|
||||
:param str resource: resource
|
||||
:rtype: bool
|
||||
:return: if resource has torrent
|
||||
|
|
@ -1072,13 +1072,13 @@ def _check_resource_has_torrent(
|
|||
|
||||
def distribute_global_resources(
|
||||
loop: asyncio.BaseEventLoop,
|
||||
blob_client: azure.storage.blob.BlockBlobService,
|
||||
table_client: azure.storage.table.TableService,
|
||||
blob_client: azureblob.BlockBlobService,
|
||||
table_client: azuretable.TableService,
|
||||
ipaddress: str) -> None:
|
||||
"""Distribute global services/resources
|
||||
:param asyncio.BaseEventLoop loop: event loop
|
||||
:param azure.storage.blob.BlockBlobService blob_client: blob client
|
||||
:param azure.storage.table.TableService table_client: table client
|
||||
:param azureblob.BlockBlobService blob_client: blob client
|
||||
:param azuretable.TableService table_client: table client
|
||||
:param str ipaddress: ip address
|
||||
"""
|
||||
# set torrent session port listen
|
||||
|
|
|
|||
|
|
@ -33,7 +33,7 @@ import pathlib
|
|||
import subprocess
|
||||
import sys
|
||||
# non-stdlib imports
|
||||
import azure.storage.table as azuretable
|
||||
import azure.cosmosdb.table as azuretable
|
||||
|
||||
# global defines
|
||||
_PARTITION_KEY = None
|
||||
|
|
@ -43,7 +43,7 @@ _TABLE_NAME = None
|
|||
def _create_credentials(config: dict) -> azuretable.TableService:
|
||||
"""Create authenticated clients
|
||||
:param dict config: configuration dict
|
||||
:rtype: azure.storage.table.TableService
|
||||
:rtype: azure.cosmosdb.table.TableService
|
||||
:return: table client
|
||||
"""
|
||||
global _PARTITION_KEY, _TABLE_NAME
|
||||
|
|
@ -168,7 +168,7 @@ def _diff_events(
|
|||
|
||||
def coalesce_data(table_client: azuretable.TableService) -> tuple:
|
||||
"""Coalesce perf data from table
|
||||
:param azure.storage.table.TableService table_client: table client
|
||||
:param azure.cosmosdb.table.TableService table_client: table client
|
||||
:rtype: tuple
|
||||
:return: (timing, sizes, offer, sku)
|
||||
"""
|
||||
|
|
|
|||
|
|
@ -30,7 +30,7 @@ import datetime
|
|||
import os
|
||||
# non-stdlib imports
|
||||
import azure.common
|
||||
import azure.storage.table as azuretable
|
||||
import azure.cosmosdb.table as azuretable
|
||||
|
||||
# global defines
|
||||
_BATCHACCOUNT = os.environ['AZ_BATCH_ACCOUNT_NAME']
|
||||
|
|
@ -41,7 +41,7 @@ _PARTITION_KEY = '{}${}'.format(_BATCHACCOUNT, _POOLID)
|
|||
|
||||
def _create_credentials() -> azuretable.TableService:
|
||||
"""Create storage credentials
|
||||
:rtype: azure.storage.table.TableService
|
||||
:rtype: azure.cosmosdb.table.TableService
|
||||
:return: azure storage table client
|
||||
"""
|
||||
sa, ep, sakey = os.environ['SHIPYARD_STORAGE_ENV'].split(':')
|
||||
|
|
@ -53,11 +53,11 @@ def _create_credentials() -> azuretable.TableService:
|
|||
|
||||
|
||||
def process_event(
|
||||
table_client: azure.storage.table.TableService,
|
||||
table_client: azuretable.TableService,
|
||||
table_name: str, source: str, event: str, ts: float,
|
||||
message: str) -> None:
|
||||
"""Process event
|
||||
:param azure.storage.table.TableService table_client: table client
|
||||
:param azure.cosmosdb.table.TableService table_client: table client
|
||||
:param str table_name: table name
|
||||
:param str source: source
|
||||
:param str event: event
|
||||
|
|
|
|||
|
|
@ -0,0 +1,2 @@
|
|||
azure-cosmosdb-table==0.37.1
|
||||
azure-storage-blob==0.37.0
|
||||
|
|
@ -34,14 +34,13 @@ import logging
|
|||
# non-stdlib imports
|
||||
import azure.batch.batch_auth as batchauth
|
||||
import azure.batch.batch_service_client as batchsc
|
||||
import azure.cosmosdb.table as azuretable
|
||||
import azure.keyvault
|
||||
import azure.mgmt.batch
|
||||
import azure.mgmt.compute
|
||||
import azure.mgmt.network
|
||||
import azure.mgmt.resource
|
||||
import azure.storage.blob as azureblob
|
||||
import azure.storage.queue as azurequeue
|
||||
import azure.storage.table as azuretable
|
||||
# local imports
|
||||
from . import aad
|
||||
from . import settings
|
||||
|
|
@ -242,7 +241,7 @@ def create_storage_clients():
|
|||
# type: (None) -> tuple
|
||||
"""Create storage clients
|
||||
:rtype: tuple
|
||||
:return: blob_client, queue_client, table_client
|
||||
:return: blob_client, table_client
|
||||
"""
|
||||
account_name = storage.get_storageaccount()
|
||||
account_key = storage.get_storageaccount_key()
|
||||
|
|
@ -252,14 +251,9 @@ def create_storage_clients():
|
|||
account_key=account_key,
|
||||
endpoint_suffix=endpoint_suffix,
|
||||
)
|
||||
queue_client = azurequeue.QueueService(
|
||||
account_name=account_name,
|
||||
account_key=account_key,
|
||||
endpoint_suffix=endpoint_suffix,
|
||||
)
|
||||
table_client = azuretable.TableService(
|
||||
account_name=account_name,
|
||||
account_key=account_key,
|
||||
endpoint_suffix=endpoint_suffix,
|
||||
)
|
||||
return blob_client, queue_client, table_client
|
||||
return blob_client, table_client
|
||||
|
|
|
|||
|
|
@ -2325,7 +2325,7 @@ def action_pool_add(
|
|||
:param azure.batch.batch_service_client.BatchServiceClient batch_client:
|
||||
batch client
|
||||
:param azure.storage.blob.BlockBlobService blob_client: blob client
|
||||
:param azure.storage.table.TableService table_client: table client
|
||||
:param azure.cosmosdb.table.TableService table_client: table client
|
||||
:param dict config: configuration dict
|
||||
"""
|
||||
_check_batch_client(batch_client)
|
||||
|
|
@ -2357,17 +2357,15 @@ def action_pool_list(batch_client):
|
|||
|
||||
|
||||
def action_pool_delete(
|
||||
batch_client, blob_client, queue_client, table_client, config,
|
||||
pool_id=None, wait=False):
|
||||
batch_client, blob_client, table_client, config, pool_id=None,
|
||||
wait=False):
|
||||
# type: (batchsc.BatchServiceClient, azureblob.BlockBlobService,
|
||||
# azurequeue.QueueService, azuretable.TableService, dict,
|
||||
# str, bool) -> None
|
||||
# azuretable.TableService, dict, str, bool) -> None
|
||||
"""Action: Pool Delete
|
||||
:param azure.batch.batch_service_client.BatchServiceClient batch_client:
|
||||
batch client
|
||||
:param azure.storage.blob.BlockBlobService blob_client: blob client
|
||||
:param azure.storage.queue.QueueService queue_client: queue client
|
||||
:param azure.storage.table.TableService table_client: table client
|
||||
:param azure.cosmosdb.table.TableService table_client: table client
|
||||
:param dict config: configuration dict
|
||||
:param str pool_id: poolid to delete
|
||||
:param bool wait: wait for pool to delete
|
||||
|
|
@ -2389,9 +2387,8 @@ def action_pool_delete(
|
|||
populate_global_settings(config, False, pool_id=pool_id)
|
||||
else:
|
||||
pool_id = settings.pool_id(config)
|
||||
# TODO remove queue_client in future release
|
||||
storage.cleanup_with_del_pool(
|
||||
blob_client, queue_client, table_client, config, pool_id=pool_id)
|
||||
blob_client, table_client, config, pool_id=pool_id)
|
||||
if wait:
|
||||
logger.debug('waiting for pool {} to delete'.format(pool_id))
|
||||
while batch_client.pool.exists(pool_id):
|
||||
|
|
@ -2747,7 +2744,7 @@ def action_jobs_add(
|
|||
:param azure.batch.batch_service_client.BatchServiceClient batch_client:
|
||||
batch client
|
||||
:param azure.storage.blob.BlockBlobService blob_client: blob client
|
||||
:param azure.storage.table.TableService table_client: table client
|
||||
:param azure.cosmosdb.table.TableService table_client: table client
|
||||
:param azure.keyvault.KeyVaultClient keyvault_client: keyvault client
|
||||
:param dict config: configuration dict
|
||||
:param bool recreate: recreate jobs if completed
|
||||
|
|
@ -2869,18 +2866,16 @@ def action_jobs_deltasks(batch_client, config, jobid, taskid, wait):
|
|||
|
||||
|
||||
def action_jobs_del_or_term(
|
||||
batch_client, blob_client, queue_client, table_client, config,
|
||||
delete, all_jobs, all_jobschedules, jobid, jobscheduleid, termtasks,
|
||||
wait):
|
||||
batch_client, blob_client, table_client, config, delete, all_jobs,
|
||||
all_jobschedules, jobid, jobscheduleid, termtasks, wait):
|
||||
# type: (batchsc.BatchServiceClient, azureblob.BlockBlobService,
|
||||
# azurequeue.QueueService, azuretable.TableService, dict, bool,
|
||||
# bool, str, str, bool, bool) -> None
|
||||
# azuretable.TableService, dict, bool, bool, str, str,
|
||||
# bool, bool) -> None
|
||||
"""Action: Jobs Del or Term
|
||||
:param azure.batch.batch_service_client.BatchServiceClient batch_client:
|
||||
batch client
|
||||
:param azure.storage.blob.BlockBlobService blob_client: blob client
|
||||
:param azure.storage.queue.QueueService queue_client: queue client
|
||||
:param azure.storage.table.TableService table_client: table client
|
||||
:param azure.cosmosdb.table.TableService table_client: table client
|
||||
:param dict config: configuration dict
|
||||
:param bool all_jobs: all jobs
|
||||
:param bool all_jobschedules: all job schedules
|
||||
|
|
@ -2928,9 +2923,7 @@ def action_jobs_del_or_term(
|
|||
jobscheduleid=jobscheduleid, termtasks=termtasks, wait=wait)
|
||||
# if autopool, delete the storage
|
||||
if autopool:
|
||||
# TODO remove queue_client in 3.0
|
||||
storage.cleanup_with_del_pool(
|
||||
blob_client, queue_client, table_client, config)
|
||||
storage.cleanup_with_del_pool(blob_client, table_client, config)
|
||||
|
||||
|
||||
def action_jobs_cmi(batch_client, config, delete):
|
||||
|
|
@ -3066,13 +3059,12 @@ def action_jobs_stats(batch_client, config, job_id):
|
|||
|
||||
|
||||
def action_storage_del(
|
||||
blob_client, queue_client, table_client, config, clear_tables, poolid):
|
||||
# type: (azureblob.BlockBlobService, azurequeue.QueueService,
|
||||
# azuretable.TableService, dict, bool, str) -> None
|
||||
blob_client, table_client, config, clear_tables, poolid):
|
||||
# type: (azureblob.BlockBlobService, azuretable.TableService,
|
||||
# dict, bool, str) -> None
|
||||
"""Action: Storage Del
|
||||
:param azure.storage.blob.BlockBlobService blob_client: blob client
|
||||
:param azure.storage.queue.QueueService queue_client: queue client
|
||||
:param azure.storage.table.TableService table_client: table client
|
||||
:param azure.cosmosdb.table.TableService table_client: table client
|
||||
:param dict config: configuration dict
|
||||
:param bool clear_tables: clear tables instead of deleting
|
||||
:param str poolid: pool id to target
|
||||
|
|
@ -3085,8 +3077,7 @@ def action_storage_del(
|
|||
blob_client, table_client, config, tables_only=True,
|
||||
pool_id=poolid)
|
||||
storage.delete_storage_containers(
|
||||
blob_client, queue_client, table_client, config,
|
||||
skip_tables=clear_tables)
|
||||
blob_client, table_client, config, skip_tables=clear_tables)
|
||||
|
||||
|
||||
def action_storage_clear(blob_client, table_client, config, poolid):
|
||||
|
|
@ -3094,7 +3085,7 @@ def action_storage_clear(blob_client, table_client, config, poolid):
|
|||
# str) -> None
|
||||
"""Action: Storage Clear
|
||||
:param azure.storage.blob.BlockBlobService blob_client: blob client
|
||||
:param azure.storage.table.TableService table_client: table client
|
||||
:param azure.cosmosdb.table.TableService table_client: table client
|
||||
:param dict config: configuration dict
|
||||
:param str poolid: pool id to target
|
||||
"""
|
||||
|
|
|
|||
|
|
@ -35,9 +35,9 @@ import hashlib
|
|||
import logging
|
||||
# non-stdlib imports
|
||||
import azure.common
|
||||
import azure.cosmosdb.table as azuretable
|
||||
import azure.storage.blob as azureblob
|
||||
import azure.storage.file as azurefile
|
||||
import azure.storage.table as azuretable
|
||||
# local imports
|
||||
from . import settings
|
||||
from . import util
|
||||
|
|
@ -62,7 +62,6 @@ _STORAGE_CONTAINERS = {
|
|||
'table_perf': None,
|
||||
# TODO remove following in future release
|
||||
'table_registry': None,
|
||||
'queue_globalresources': None,
|
||||
}
|
||||
|
||||
|
||||
|
|
@ -94,8 +93,6 @@ def set_storage_configuration(sep, postfix, sa, sakey, saep, sasexpiry):
|
|||
_STORAGE_CONTAINERS['table_perf'] = sep + 'perf'
|
||||
# TODO remove following containers in future release
|
||||
_STORAGE_CONTAINERS['table_registry'] = sep + 'registry'
|
||||
_STORAGE_CONTAINERS['queue_globalresources'] = '-'.join(
|
||||
(sep + 'gr', postfix))
|
||||
# ensure all storage containers are between 3 and 63 chars in length
|
||||
for key in _STORAGE_CONTAINERS:
|
||||
length = len(_STORAGE_CONTAINERS[key])
|
||||
|
|
@ -246,11 +243,11 @@ def _construct_partition_key_from_config(config, pool_id=None):
|
|||
|
||||
def _add_global_resource(
|
||||
blob_client, table_client, config, pk, dr, grtype):
|
||||
# type: (azurequeue.QueueService, azuretable.TableService, dict, str,
|
||||
# type: (azureblob.BlockBlobService, azuretable.TableService, dict, str,
|
||||
# settings.DataReplicationSettings, str) -> None
|
||||
"""Add global resources
|
||||
:param azure.storage.blob.BlockService blob_client: blob client
|
||||
:param azure.storage.table.TableService table_client: table client
|
||||
:param azure.cosmosdb.table.TableService table_client: table client
|
||||
:param dict config: configuration dict
|
||||
:param str pk: partition key
|
||||
:param settings.DataReplicationSettings dr: data replication settings
|
||||
|
|
@ -288,10 +285,10 @@ def _add_global_resource(
|
|||
|
||||
|
||||
def populate_global_resource_blobs(blob_client, table_client, config):
|
||||
# type: (azureblob.BlockService, azuretable.TableService, dict) -> None
|
||||
# type: (azureblob.BlockBlobService, azuretable.TableService, dict) -> None
|
||||
"""Populate global resource blobs
|
||||
:param azure.storage.blob.BlockService blob_client: blob client
|
||||
:param azure.storage.table.TableService table_client: table client
|
||||
:param azure.cosmosdb.table.TableService table_client: table client
|
||||
:param dict config: configuration dict
|
||||
"""
|
||||
pk = _construct_partition_key_from_config(config)
|
||||
|
|
@ -372,22 +369,17 @@ def upload_for_remotefs(blob_client, files):
|
|||
|
||||
|
||||
def delete_storage_containers(
|
||||
blob_client, queue_client, table_client, config, skip_tables=False):
|
||||
# type: (azureblob.BlockBlobService, azurequeue.QueueService,
|
||||
# azuretable.TableService, dict, bool) -> None
|
||||
blob_client, table_client, config, skip_tables=False):
|
||||
# type: (azureblob.BlockBlobService, azuretable.TableService,
|
||||
# dict, bool) -> None
|
||||
"""Delete storage containers
|
||||
:param azure.storage.blob.BlockBlobService blob_client: blob client
|
||||
:param azure.storage.queue.QueueService queue_client: queue client
|
||||
:param azure.storage.table.TableService table_client: table client
|
||||
:param azure.cosmosdb.table.TableService table_client: table client
|
||||
:param dict config: configuration dict
|
||||
:param bool skip_tables: skip deleting tables
|
||||
"""
|
||||
for key in _STORAGE_CONTAINERS:
|
||||
if key.startswith('queue_'):
|
||||
# TODO remove in future release: unused queues
|
||||
logger.debug('deleting queue: {}'.format(_STORAGE_CONTAINERS[key]))
|
||||
queue_client.delete_queue(_STORAGE_CONTAINERS[key])
|
||||
elif key == 'table_registry':
|
||||
if key == 'table_registry':
|
||||
# TODO remove in future release: unused table
|
||||
logger.debug('deleting table: {}'.format(_STORAGE_CONTAINERS[key]))
|
||||
table_client.delete_table(_STORAGE_CONTAINERS[key])
|
||||
|
|
@ -439,7 +431,7 @@ def _clear_blob_task_resourcefiles(blob_client, container, config):
|
|||
def _clear_table(table_client, table_name, config, pool_id=None):
|
||||
# type: (azuretable.TableService, str, dict, str) -> None
|
||||
"""Clear table entities
|
||||
:param azure.storage.table.TableService table_client: table client
|
||||
:param azure.cosmosdb.table.TableService table_client: table client
|
||||
:param str table_name: table name
|
||||
:param dict config: configuration dict
|
||||
:param str pool_id: use specified pool id instead
|
||||
|
|
@ -468,7 +460,7 @@ def clear_storage_containers(
|
|||
# bool, str) -> None
|
||||
"""Clear storage containers
|
||||
:param azure.storage.blob.BlockBlobService blob_client: blob client
|
||||
:param azure.storage.table.TableService table_client: table client
|
||||
:param azure.cosmosdb.table.TableService table_client: table client
|
||||
:param dict config: configuration dict
|
||||
:param bool tables_only: clear only tables
|
||||
:param str pool_id: use specified pool id instead
|
||||
|
|
@ -495,7 +487,7 @@ def create_storage_containers(blob_client, table_client, config):
|
|||
# type: (azureblob.BlockBlobService, azuretable.TableService, dict) -> None
|
||||
"""Create storage containers
|
||||
:param azure.storage.blob.BlockBlobService blob_client: blob client
|
||||
:param azure.storage.table.TableService table_client: table client
|
||||
:param azure.cosmosdb.table.TableService table_client: table client
|
||||
:param dict config: configuration dict
|
||||
"""
|
||||
bs = settings.batch_shipyard_settings(config)
|
||||
|
|
@ -534,14 +526,12 @@ def delete_storage_containers_remotefs(blob_client):
|
|||
blob_client.delete_container(contname)
|
||||
|
||||
|
||||
def cleanup_with_del_pool(
|
||||
blob_client, queue_client, table_client, config, pool_id=None):
|
||||
# type: (azureblob.BlockBlobService, azurequeue.QueueService,
|
||||
# azuretable.TableService, dict, str) -> None
|
||||
def cleanup_with_del_pool(blob_client, table_client, config, pool_id=None):
|
||||
# type: (azureblob.BlockBlobService, azuretable.TableService,
|
||||
# dict, str) -> None
|
||||
"""Special cleanup routine in combination with delete pool
|
||||
:param azure.storage.blob.BlockBlobService blob_client: blob client
|
||||
:param azure.storage.queue.QueueService queue_client: queue client
|
||||
:param azure.storage.table.TableService table_client: table client
|
||||
:param azure.cosmosdb.table.TableService table_client: table client
|
||||
:param dict config: configuration dict
|
||||
:param str pool_id: pool id
|
||||
"""
|
||||
|
|
@ -554,4 +544,4 @@ def cleanup_with_del_pool(
|
|||
clear_storage_containers(
|
||||
blob_client, table_client, config, tables_only=True, pool_id=pool_id)
|
||||
delete_storage_containers(
|
||||
blob_client, queue_client, table_client, config, skip_tables=True)
|
||||
blob_client, table_client, config, skip_tables=True)
|
||||
|
|
|
|||
|
|
@ -1,15 +1,17 @@
|
|||
adal==0.4.7
|
||||
azure-batch==4.0.0
|
||||
azure-cosmosdb-table==0.37.1
|
||||
azure-keyvault==0.3.7
|
||||
azure-mgmt-batch==4.1.0
|
||||
azure-mgmt-compute==1.0.0
|
||||
azure-mgmt-network==1.5.0
|
||||
azure-mgmt-resource==1.1.0
|
||||
azure-storage==0.36.0
|
||||
blobxfer==1.0.0b2
|
||||
azure-storage-blob==0.37.0
|
||||
azure-storage-file==0.37.0
|
||||
blobxfer==1.0.0rc1
|
||||
click==6.7
|
||||
future==0.16.0
|
||||
msrest==0.4.15
|
||||
msrest==0.4.16
|
||||
msrestazure==0.4.14
|
||||
pathlib2==2.3.0; python_version < '3.5'
|
||||
python-dateutil==2.6.1
|
||||
|
|
|
|||
|
|
@ -617,7 +617,9 @@ if [ $offer == "ubuntuserver" ] || [ $offer == "debian" ]; then
|
|||
if [ $cascadecontainer -eq 0 ]; then
|
||||
# install azure storage python dependency
|
||||
install_packages $offer build-essential libssl-dev libffi-dev libpython3-dev python3-dev python3-pip
|
||||
pip3 install --no-cache-dir azure-storage==0.36.0
|
||||
pip3 install --no-cache-dir --upgrade pip
|
||||
pip3 install --no-cache-dir --upgrade wheel setuptools
|
||||
pip3 install --no-cache-dir azure-cosmosdb-table==0.37.1 azure-storage-blob==0.37.0
|
||||
# install cascade dependencies
|
||||
if [ $p2penabled -eq 1 ]; then
|
||||
install_packages $offer python3-libtorrent pigz
|
||||
|
|
|
|||
42
shipyard.py
42
shipyard.py
|
|
@ -64,7 +64,6 @@ class CliContext(object):
|
|||
self.batch_mgmt_client = None
|
||||
self.batch_client = None
|
||||
self.blob_client = None
|
||||
self.queue_client = None
|
||||
self.table_client = None
|
||||
self.keyvault_client = None
|
||||
self.resource_client = None
|
||||
|
|
@ -96,7 +95,7 @@ class CliContext(object):
|
|||
skip_global_config=False, skip_pool_config=True, fs_storage=True)
|
||||
self.resource_client, self.compute_client, self.network_client, \
|
||||
_, _ = convoy.clients.create_arm_clients(self)
|
||||
self.blob_client, _, _ = convoy.clients.create_storage_clients()
|
||||
self.blob_client, _ = convoy.clients.create_storage_clients()
|
||||
self._cleanup_after_initialize(
|
||||
skip_global_config=False, skip_pool_config=True)
|
||||
|
||||
|
|
@ -126,7 +125,7 @@ class CliContext(object):
|
|||
self.resource_client, self.compute_client, self.network_client, \
|
||||
self.batch_mgmt_client, self.batch_client = \
|
||||
convoy.clients.create_arm_clients(self, batch_clients=True)
|
||||
self.blob_client, self.queue_client, self.table_client = \
|
||||
self.blob_client, self.table_client = \
|
||||
convoy.clients.create_storage_clients()
|
||||
self._cleanup_after_initialize(
|
||||
skip_global_config=False, skip_pool_config=False)
|
||||
|
|
@ -141,7 +140,7 @@ class CliContext(object):
|
|||
self.keyvault_client = convoy.clients.create_keyvault_client(self)
|
||||
self._init_config(
|
||||
skip_global_config=False, skip_pool_config=False, fs_storage=False)
|
||||
self.blob_client, self.queue_client, self.table_client = \
|
||||
self.blob_client, self.table_client = \
|
||||
convoy.clients.create_storage_clients()
|
||||
self._cleanup_after_initialize(
|
||||
skip_global_config=False, skip_pool_config=False)
|
||||
|
|
@ -335,16 +334,12 @@ class CliContext(object):
|
|||
# show config if specified
|
||||
if self.show_config:
|
||||
logger.debug('config:\n' + json.dumps(self.config, indent=4))
|
||||
|
||||
def _set_clients(
|
||||
self, batch_mgmt_client, batch_client, blob_client, queue_client,
|
||||
table_client):
|
||||
"""Sets clients for the context"""
|
||||
self.batch_mgmt_client = batch_mgmt_client
|
||||
self.batch_client = batch_client
|
||||
self.blob_client = blob_client
|
||||
self.queue_client = queue_client
|
||||
self.table_client = table_client
|
||||
# disable azure storage/cosmosdb logging: setting logger level
|
||||
# to CRITICAL effectively disables logging from azure storage/cosmosdb
|
||||
az_logger = logging.getLogger('azure.storage')
|
||||
az_logger.setLevel(logging.CRITICAL)
|
||||
az_logger = logging.getLogger('azure.cosmosdb')
|
||||
az_logger.setLevel(logging.CRITICAL)
|
||||
|
||||
|
||||
# create a pass decorator for shared context between commands
|
||||
|
|
@ -928,8 +923,7 @@ def storage_del(ctx, clear_tables, poolid):
|
|||
"""Delete Azure Storage containers used by Batch Shipyard"""
|
||||
ctx.initialize_for_storage()
|
||||
convoy.fleet.action_storage_del(
|
||||
ctx.blob_client, ctx.queue_client, ctx.table_client, ctx.config,
|
||||
clear_tables, poolid)
|
||||
ctx.blob_client, ctx.table_client, ctx.config, clear_tables, poolid)
|
||||
|
||||
|
||||
@storage.command('clear')
|
||||
|
|
@ -1106,8 +1100,8 @@ def pool_del(ctx, poolid, wait):
|
|||
"""Delete a pool from the Batch account"""
|
||||
ctx.initialize_for_batch()
|
||||
convoy.fleet.action_pool_delete(
|
||||
ctx.batch_client, ctx.blob_client, ctx.queue_client,
|
||||
ctx.table_client, ctx.config, pool_id=poolid, wait=wait)
|
||||
ctx.batch_client, ctx.blob_client, ctx.table_client, ctx.config,
|
||||
pool_id=poolid, wait=wait)
|
||||
|
||||
|
||||
@pool.command('resize')
|
||||
|
|
@ -1451,9 +1445,9 @@ def jobs_term(
|
|||
"""Terminate jobs and job schedules"""
|
||||
ctx.initialize_for_batch()
|
||||
convoy.fleet.action_jobs_del_or_term(
|
||||
ctx.batch_client, ctx.blob_client, ctx.queue_client, ctx.table_client,
|
||||
ctx.config, False, all_jobs, all_jobschedules, jobid, jobscheduleid,
|
||||
termtasks, wait)
|
||||
ctx.batch_client, ctx.blob_client, ctx.table_client, ctx.config,
|
||||
False, all_jobs, all_jobschedules, jobid, jobscheduleid, termtasks,
|
||||
wait)
|
||||
|
||||
|
||||
@jobs.command('del')
|
||||
|
|
@ -1481,9 +1475,9 @@ def jobs_del(
|
|||
"""Delete jobs and job schedules"""
|
||||
ctx.initialize_for_batch()
|
||||
convoy.fleet.action_jobs_del_or_term(
|
||||
ctx.batch_client, ctx.blob_client, ctx.queue_client, ctx.table_client,
|
||||
ctx.config, True, all_jobs, all_jobschedules, jobid, jobscheduleid,
|
||||
termtasks, wait)
|
||||
ctx.batch_client, ctx.blob_client, ctx.table_client, ctx.config,
|
||||
True, all_jobs, all_jobschedules, jobid, jobscheduleid, termtasks,
|
||||
wait)
|
||||
|
||||
|
||||
@jobs.command('deltasks')
|
||||
|
|
|
|||
Загрузка…
Ссылка в новой задаче