diff --git a/.dockerignore b/.dockerignore index 5d401994..feabe66d 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,6 +1,5 @@ .ignore Dockerfile -README.md maro_cli/ examples/**/*.log examples/**/*.csv diff --git a/.github/workflows/build_wheel.yml b/.github/workflows/build_wheel.yml index dd379542..35b7f75b 100644 --- a/.github/workflows/build_wheel.yml +++ b/.github/workflows/build_wheel.yml @@ -29,7 +29,7 @@ jobs: with: python-version: ${{ matrix.python-version }} - - name: install dependencies + - name: Install dependencies run: | python -m pip install --upgrade pip pip install wheel twine @@ -42,12 +42,12 @@ jobs: run: | cython ./maro/backends/backend.pyx ./maro/backends/np_backend.pyx ./maro/backends/raw_backend.pyx ./maro/backends/frame.pyx -3 -E FRAME_BACKEND=NUMPY,NODES_MEMORY_LAYOUT=ONE_BLOCK -X embedsignature=True - - name: build wheel on Windows and macOS + - name: Build wheel on Windows and macOS if: runner.os == 'Windows' || runner.os == 'macOS' run: | python setup.py bdist_wheel - - name: build manylinux wheel + - name: Build manylinux wheel if: runner.os == 'Linux' && matrix.python-version == '3.6' uses: RalfG/python-wheels-manylinux-build@v0.3.1-manylinux2010_x86_64 with: @@ -55,13 +55,13 @@ jobs: build-requirements: 'numpy' pip-wheel-args: '-w ./wheelhouse' # save wheel packages to wheelhouse folder - - name: move valid packages to dist folder for manylinux + - name: Move valid packages to dist folder for manylinux if: runner.os == 'Linux' && matrix.python-version == '3.6' run: | mkdir -p dist cp wheelhouse/maro-*-manylinux*.whl dist - - name: build source package on linux + - name: Build source package on linux if: runner.os == 'Linux' && matrix.python-version == '3.6' run: | python setup.py sdist diff --git a/.github/workflows/deploy_docker_image.yml b/.github/workflows/deploy_docker_image.yml index a98f8949..6052457a 100644 --- a/.github/workflows/deploy_docker_image.yml +++ b/.github/workflows/deploy_docker_image.yml @@ -25,30 +25,23 @@ jobs: # Steps represent a sequence of tasks that will be executed as part of the job steps: - # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it - - uses: actions/checkout@v2 + # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it + - uses: actions/checkout@v2 + - name: Set up Python 3.6 + uses: actions/setup-python@v2 + with: + python-version: 3.6 - - name: Build and push Docker images - # You may pin to the exact commit or the version. - # uses: docker/build-push-action@ab83648e2e224cfeeab899e23b639660765c3a89 - uses: docker/build-push-action@v1.1.1 - with: - # Username used to log in to a Docker registry. If not set then no login will occur - username: ${{ secrets.DOCKER_HUB_USERNAME }} - # Password or personal access token used to log in to a Docker registry. If not set then no login will occur - password: ${{ secrets.DOCKER_HUB_PASSWORD }} - # Server address of Docker registry. If not set then will default to Docker Hub - registry: https://hub.docker.com/repository/docker/arthursjiang/maro - # Docker repository to tag the image with - repository: latest - # Comma-delimited list of tags. These will be added to the registry/repository to form the image's tags - tags: citi-bike, inventory-management, operations-research, reinforcement-learning, resource-optimization, simulator - # Automatically tags the built image with the git reference as per the readme - # Path to the build context - path: . - # Path to the Dockerfile (Default is '{path}/Dockerfile') - dockerfile: docker_files/cpu.play.df - # Sets the target stage to build - target: maro/playground:cpu - # Whether to push the image - push: true + - name: Build image + run: | + pip install -r ./maro/requirements.build.txt + cython ./maro/backends/backend.pyx ./maro/backends/np_backend.pyx ./maro/backends/raw_backend.pyx ./maro/backends/frame.pyx -3 -E FRAME_BACKEND=NUMPY,NODES_MEMORY_LAYOUT=ONE_BLOCK -X embedsignature=True + docker build -f ./docker_files/cpu.play.df . -t ${{ secrets.DOCKER_HUB_USERNAME }}/maro:cpu + + - name: Login docker hub + run: | + docker login --username ${{ secrets.DOCKER_HUB_USERNAME }} --password ${{ secrets.DOCKER_HUB_PASSWORD }} + + - name: Push image + run: | + docker push ${{ secrets.DOCKER_HUB_USERNAME }}/maro diff --git a/.github/workflows/deploy_gh_pages.yml b/.github/workflows/deploy_gh_pages.yml index 7abb8717..39f31cd1 100644 --- a/.github/workflows/deploy_gh_pages.yml +++ b/.github/workflows/deploy_gh_pages.yml @@ -14,7 +14,7 @@ on: logLevel: description: 'package version' required: true - default: '0.0.1a0' + default: '0.1.1a0' jobs: build: diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 00000000..5b8e2e37 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,13 @@ +# Contributing to MARO + +MARO is newborn for Reinforcement learning as a Service (RaaS) in the resource optimization domain. Your contribution is precious to make RaaS come true. + +- [Open issues](https://github.com/microsoft/maro/issues) for reporting bugs and requesting new features. +- Contribute to [examples](https://github.com/microsoft/maro/tree/master/examples) to share your problem modeling to others. +- Contribute to [scenarios](https://github.com/microsoft/maro/tree/master/maro/simulator/scenarios) to provide more meaningful environments. +- Contribute to [topologies](https://github.com/microsoft/maro/tree/master/maro/simulator/scenarios/citi_bike/topologies) to enhance existing MARO scenarios. +- Contribute to [algorithms](https://github.com/microsoft/maro/tree/master/maro/rl/algorithms) to enrich MARO RL libraries. +- Contribute to [orchestration](https://github.com/microsoft/maro/tree/master/maro/cli) to broad MARO supported cloud services. +- Contribute to [communication](https://github.com/microsoft/maro/tree/master/maro/communication) to enhance MARO distributed training capacity. +- Contribute to [tests](https://github.com/microsoft/maro/tree/master/tests) to make it more reliable and stable. +- Contribute to [documentation](https://github.com/microsoft/maro/tree/master/maro) to make it straightforward for everyone. \ No newline at end of file diff --git a/docker_files/cpu.play.df b/docker_files/cpu.play.df index d4f73ea7..2bc4a9dc 100644 --- a/docker_files/cpu.play.df +++ b/docker_files/cpu.play.df @@ -4,6 +4,8 @@ FROM quay.io/pypa/manylinux2010_x86_64 as ext_build WORKDIR /maro_build ADD ./maro ./maro +ADD README.md ./README.md +ADD LICENSE ./LICENSE ADD setup.py ./setup.py RUN PYBIN="/opt/python/cp36-cp36m/bin"; "${PYBIN}/pip" install -r maro/requirements.build.txt; "${PYBIN}/python" setup.py bdist_wheel --plat-name manylinux2010_x86_64 diff --git a/docker_files/cpu.runtime.df b/docker_files/cpu.runtime.df index 398a958c..b4c36c7b 100644 --- a/docker_files/cpu.runtime.df +++ b/docker_files/cpu.runtime.df @@ -6,6 +6,8 @@ WORKDIR /maro_build # Build wheels ADD ./maro ./maro ADD setup.py ./setup.py +ADD README.md ./README.md +ADD LICENSE ./LICENSE RUN PYBIN="/opt/python/cp36-cp36m/bin"; "${PYBIN}/pip" install -r maro/requirements.build.txt; "${PYBIN}/python" setup.py bdist_wheel --plat-name manylinux2010_x86_64 @@ -36,6 +38,6 @@ RUN pip install psutil==5.7.2 RUN pip install deepdiff==5.0.2 # Install maro -COPY --from=ext_build /maro_build/dist/maro-0.0.1a0-cp36-cp36m-manylinux2010_x86_64.whl ./maro-0.0.1a0-cp36-cp36m-manylinux2010_x86_64.whl -RUN pip install maro-0.0.1a0-cp36-cp36m-manylinux2010_x86_64.whl -RUN rm maro-0.0.1a0-cp36-cp36m-manylinux2010_x86_64.whl +COPY --from=ext_build /maro_build/dist/maro-0.1.1a0-cp36-cp36m-manylinux2010_x86_64.whl ./maro-0.1.1a0-cp36-cp36m-manylinux2010_x86_64.whl +RUN pip install maro-0.1.1a0-cp36-cp36m-manylinux2010_x86_64.whl +RUN rm maro-0.1.1a0-cp36-cp36m-manylinux2010_x86_64.whl diff --git a/docker_files/gpu.runtime.df b/docker_files/gpu.runtime.df index 1adc67f3..56865660 100644 --- a/docker_files/gpu.runtime.df +++ b/docker_files/gpu.runtime.df @@ -6,6 +6,8 @@ WORKDIR /maro_build # Build wheels ADD ./maro ./maro ADD setup.py ./setup.py +ADD README.md ./README.md +ADD LICENSE ./LICENSE RUN PYBIN="/opt/python/cp36-cp36m/bin"; "${PYBIN}/pip" install -r maro/requirements.build.txt; "${PYBIN}/python" setup.py bdist_wheel --plat-name manylinux2010_x86_64 @@ -41,6 +43,6 @@ RUN pip install psutil==5.7.2 RUN pip install deepdiff==5.0.2 # Install maro -COPY --from=ext_build /maro_build/dist/maro-0.0.1a0-cp36-cp36m-manylinux2010_x86_64.whl ./maro-0.0.1a0-cp36-cp36m-manylinux2010_x86_64.whl -RUN pip install maro-0.0.1a0-cp36-cp36m-manylinux2010_x86_64.whl -RUN rm maro-0.0.1a0-cp36-cp36m-manylinux2010_x86_64.whl +COPY --from=ext_build /maro_build/dist/maro-0.1.1a0-cp36-cp36m-manylinux2010_x86_64.whl ./maro-0.1.1a0-cp36-cp36m-manylinux2010_x86_64.whl +RUN pip install maro-0.1.1a0-cp36-cp36m-manylinux2010_x86_64.whl +RUN rm maro-0.1.1a0-cp36-cp36m-manylinux2010_x86_64.whl diff --git a/docs/source/apidoc/maro.rst b/docs/source/apidoc/maro.rst index 10cf7fb0..bc0ce784 100644 --- a/docs/source/apidoc/maro.rst +++ b/docs/source/apidoc/maro.rst @@ -1,4 +1,4 @@ -maro package +MARO Package ============ Subpackages @@ -13,7 +13,7 @@ Subpackages maro.simulator maro.utils -Module contents +Module Contents --------------- .. automodule:: maro diff --git a/docs/source/conf.py b/docs/source/conf.py index 63eb749c..cc083629 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -20,10 +20,11 @@ import sys from recommonmark.parser import CommonMarkParser sys.path.insert(0, os.path.abspath('../..')) -os.environ["APIDOC_GEN"] = os.environ.get("APIDOC_GEN") or "True" +# TODO: improve API Docs +os.environ["APIDOC_GEN"] = os.environ.get("APIDOC_GEN", "False") # -- Project information ----------------------------------------------------- -project = 'maro' +project = 'MARO' copyright = '2020 Microsoft' author = 'MARO Team' diff --git a/docs/source/images/distributed/key_components.svg b/docs/source/images/distributed/key_components.svg index 21b1b503..741f2a64 100644 --- a/docs/source/images/distributed/key_components.svg +++ b/docs/source/images/distributed/key_components.svg @@ -1,3 +1,3 @@ -
Proxy
Proxy
Conditional Event Register Table
Conditional Event Regis...
Cluster Management
Cluster Management
Job Management
Job Management
Communication
Communication
Orchestration
Orchestration
Distributed Toolkit
Distributed Toolkit
Data Deployment
Data Deployment
...
...
...
...Viewer does not support full SVG 1.1 \ No newline at end of file +
Proxy
Proxy
Conditional Event Register Table
Conditional Event R...
Cluster Management
Cluster Management
Job Management
Job Management
Communication
Communication
Orchestration
Orchestration
Distributed Toolkit
Distributed Toolkit
Data Deployment
Data Deployment
...
...
...
...Viewer does not support full SVG 1.1 \ No newline at end of file diff --git a/docs/source/images/distributed/overview.svg b/docs/source/images/distributed/overview.svg index f1ac29ad..ce60271e 100644 --- a/docs/source/images/distributed/overview.svg +++ b/docs/source/images/distributed/overview.svg @@ -1,3 +1,3 @@ -
Redis
Redis
Register Table
Register Table
Proxy
Proxy
Register Table
Register Table
Proxy
Proxy
Proxy
Proxy
Master Component
Master Component
Orchestration
Orchestration
Container
Container
Container
Container
Container
Container
...
...
...
...
launch
launch
launch
launch
launch
launch
discovery peers
discovery peers
discovery peers
discovery peers
discovery peers
discovery peers
dispatch task/return result
dispatch task/return resu...
send/receive message
send/receive me...
dispatch task/return result
dispatch task/return res...
send/receive message
send/receive me...
deliver message
deliver message
send task/receive result
send task/receive res...
deliver message
deliver message
Worker Component
Worker Component
Worker Component
Worker ComponentViewer does not support full SVG 1.1 \ No newline at end of file +
Redis
Redis
Register Table
Register Table
Proxy
Proxy
Register Table
Register Table
Proxy
Proxy
Proxy
Proxy
Master Component
Master Component
Orchestration
Orchestration
Container
Container
Container
Container
Container
Container
...
...
...
...
launch
launch
launch
launch
launch
launch
discovery peers
discovery peers
discovery peers
discovery peers
discovery peers
discovery peers
dispatch task/return result
dispatch task/return res...
send/receive message
send/receive...
dispatch task/return result
dispatch task/return res...
send/receive message
send/receive...
deliver message
deliver message
send task/receive result
send task/receive resu...
deliver message
deliver message
Worker Component
Worker Component
Worker Component
Worker ComponentViewer does not support full SVG 1.1 \ No newline at end of file diff --git a/docs/source/images/distributed/register_table.register.svg b/docs/source/images/distributed/register_table.register.svg new file mode 100644 index 00000000..57926b54 --- /dev/null +++ b/docs/source/images/distributed/register_table.register.svg @@ -0,0 +1,3 @@ + + +
Conditional Event
Conditional Event
Conditional Event
Conditional Event
Conditional Event
Conditional Event
Handler Function
Handler Function
Handler Function
Handler Function
Handler Function
Handler Function
Register Table
Register Table
register
regist...
Register conditional event and related handler function
Register conditional event and related handler fun...Viewer does not support full SVG 1.1 \ No newline at end of file diff --git a/docs/source/images/distributed/register_table.trigger.svg b/docs/source/images/distributed/register_table.trigger.svg new file mode 100644 index 00000000..b29add99 --- /dev/null +++ b/docs/source/images/distributed/register_table.trigger.svg @@ -0,0 +1,3 @@ + + +
Conditional Event
Conditional Event
Conditional Event
Conditional Event
Conditional Event
Conditional Event
Handler Function
Handler Function
Handler Function
Handler Function
Handler Function
Handler Function
Register Table
Register Table
Conditional Event
Conditional Event
Conditional Event
Conditional Event
Conditional Event
Conditional Event
Handler Function
Handler Function
Handler Function
Handler Function
Handler Function
Handler Function
Register Table
Register Table
Dispatch the message to the related conditional event
Dispatch the message to the related conditional ev...
Invoke the related handler function, when condition of the event is fulfilled
Invoke the related handler function, when condition of the event is...
new
message
new...
trigger
triggerViewer does not support full SVG 1.1 \ No newline at end of file diff --git a/docs/source/images/rl/agent.svg b/docs/source/images/rl/agent.svg index 903f73a2..2fe93300 100644 --- a/docs/source/images/rl/agent.svg +++ b/docs/source/images/rl/agent.svg @@ -1,3 +1,3 @@ -
Algorithm
Algorithm
Experience Pool
Experience Pool
Agent
(Inference Mode)
Agent...
State
State
Action
Acti...
Experiences
Experiences
Algorithm
Algorithm
Experience Pool
Experience Pool
Agent
(Training Mode)
Agent...
Train
Train
Samples
Samples
Models
ModelsViewer does not support full SVG 1.1 \ No newline at end of file +
Algorithm
Algorithm
Experience Pool
Experience Pool
Agent
(Inference Mode)
Agent...
State
State
Action
Acti...
Experiences
Experiences
Algorithm
Algorithm
Experience Pool
Experience Pool
Agent
(Training Mode)
Agent...
Train
Train
Samples
Samples
Models
Models
(1) Inference Mode
(1) Inference Mode
(2) Training Mode
(2) Training ModeViewer does not support full SVG 1.1 \ No newline at end of file diff --git a/docs/source/images/rl/agent_manager.svg b/docs/source/images/rl/agent_manager.svg index 69e68463..bbc65120 100644 --- a/docs/source/images/rl/agent_manager.svg +++ b/docs/source/images/rl/agent_manager.svg @@ -1,3 +1,3 @@ -
Agent
Agent
Agent
Agent
%3CmxGraphModel%3E%3Croot%3E%3CmxCell%20id%3D%220%22%2F%3E%3CmxCell%20id%3D%221%22%20parent%3D%220%22%2F%3E%3CmxCell%20id%3D%222%22%20value%3D%22%22%20style%3D%22curved%3D1%3BendArrow%3Dclassic%3Bhtml%3D1%3BfontSize%3D15%3B%22%20edge%3D%221%22%20parent%3D%221%22%3E%3CmxGeometry%20width%3D%2250%22%20height%3D%2250%22%20relative%3D%221%22%20as%3D%22geometry%22%3E%3CmxPoint%20x%3D%22410%22%20y%3D%22720%22%20as%3D%22sourcePoint%22%2F%3E%3CmxPoint%20x%3D%22460%22%20y%3D%22670%22%20as%3D%22targetPoint%22%2F%3E%3CArray%20as%3D%22points%22%3E%3CmxPoint%20x%3D%22460%22%20y%3D%22720%22%2F%3E%3CmxPoint%20x%3D%22410%22%20y%3D%22670%22%2F%3E%3C%2FArray%3E%3C%2FmxGeometry%3E%3C%2FmxCell%3E%3C%2Froot%3E%3C%2FmxGraphModel%3E
%3C...
State Shaper
State Shaper
Agent
Agent
Action Shaper
Action Shaper
Experience Shaper
Experience Shaper
Model State
Model State
Experiences
Experiences
Transitions
Transitions
Model Action
Model Action
Agent Manager
(Inference Mode)
Agent Manager...
Decision Event
Snapshot List
Decision Event...
Environment Action
Environment Action
Agent
Agent
Agent
Agent
Agent
Agent
Train
Train
Agent Manager
(Training Mode)
Agent Manager...
Models
ModelsViewer does not support full SVG 1.1 \ No newline at end of file +
Agent
Agent
Agent
Agent
Agent
Agent
Train
Train
Agent Manager
(Training Mode)
Agent Manager...
Models
Models
Agent
Agent
Agent
Agent
%3CmxGraphModel%3E%3Croot%3E%3CmxCell%20id%3D%220%22%2F%3E%3CmxCell%20id%3D%221%22%20parent%3D%220%22%2F%3E%3CmxCell%20id%3D%222%22%20value%3D%22%22%20style%3D%22curved%3D1%3BendArrow%3Dclassic%3Bhtml%3D1%3BfontSize%3D15%3B%22%20edge%3D%221%22%20parent%3D%221%22%3E%3CmxGeometry%20width%3D%2250%22%20height%3D%2250%22%20relative%3D%221%22%20as%3D%22geometry%22%3E%3CmxPoint%20x%3D%22410%22%20y%3D%22720%22%20as%3D%22sourcePoint%22%2F%3E%3CmxPoint%20x%3D%22460%22%20y%3D%22670%22%20as%3D%22targetPoint%22%2F%3E%3CArray%20as%3D%22points%22%3E%3CmxPoint%20x%3D%22460%22%20y%3D%22720%22%2F%3E%3CmxPoint%20x%3D%22410%22%20y%3D%22670%22%2F%3E%3C%2FArray%3E%3C%2FmxGeometry%3E%3C%2FmxCell%3E%3C%2Froot%3E%3C%2FmxGraphModel%3E
%3C...
State Shaper
State Shaper
Agent
Agent
Action Shaper
Action Shaper
Experience Shaper
Experience Shaper
Model State
Model State
Experiences
Experiences
Transitions
Transitions
Model Action
Model Action
Agent Manager
(Inference Mode)
Agent Manager...
Decision Event
Snapshot List
Decision Event...
Environment Action
Environment Action
(1) Inference Mode
(1) Inference Mode
(2) Training Mode
(2) Training ModeViewer does not support full SVG 1.1 \ No newline at end of file diff --git a/docs/source/images/scenario/cim.container_flow.svg b/docs/source/images/scenario/cim.container_flow.svg index 098fd81c..4ce03d4e 100644 --- a/docs/source/images/scenario/cim.container_flow.svg +++ b/docs/source/images/scenario/cim.container_flow.svg @@ -1,3 +1,3 @@ -
release empty
release empty
load full
load full
load/discharge empty
load/discharge empty
Source Port
Source Port
return full
return full
Shipper
Shipper
send order
send order
load/discharge empty
load/discharge empty
Destination Port
Destination Port
return empty
return empty
Consignee
Consignee
release full
release full
discharge full
discharge full
Vessel
Vessel
operate empty
operate empty
Agent
Agent
operate empty
operate empty
Agent
AgentViewer does not support full SVG 1.1 \ No newline at end of file +
release empty
release empty
load laden
load laden
load/discharge empty
load/discharge empty
Source Port
Source Port
return laden
return laden
Shipper
Shipper
send order
send order
load/discharge empty
load/discharge empty
Destination Port
Destination Port
return empty
return empty
Consignee
Consignee
release laden
release laden
discharge
laden
discharge...
Vessel
Vessel
Agent
Agent
operate
empty
operate...
Agent
Agent
operate
empty
operate...Viewer does not support full SVG 1.1 \ No newline at end of file diff --git a/docs/source/images/simulator/business_engine.svg b/docs/source/images/simulator/business_engine.svg index aef3d57d..463b4c45 100644 --- a/docs/source/images/simulator/business_engine.svg +++ b/docs/source/images/simulator/business_engine.svg @@ -1,3 +1,3 @@ -
Data Model
Data Model
Event Buffer
Event Buffer
Environment
Environment
Business Engine
Business Engine
Scenario-Specific Business Engine
Scenario-Specific Business Engine
Business Time Series Data
Business Time Series Data
Generate business events
Generate business...
Define/register business event and handler
Define/register business event and handl...
Define business instance
Define business instance
Load
LoadViewer does not support full SVG 1.1 \ No newline at end of file +
Data Model
Data Model
Event Buffer
Event Buffer
Environment
Environment
Business Engine
Business Engine
Scenario-Specific Business Engine
Scenario-Specific Business Engine
Business Time Series Data
Business Time Series Data
Generate
business events
Generate...
Define/register business event and handler
Define/register business event and hand...
Define business instance
Define business instance
Load
LoadViewer does not support full SVG 1.1 \ No newline at end of file diff --git a/docs/source/key_components/communication.md b/docs/source/key_components/communication.md index 6f1c0d4e..d50fd65e 100644 --- a/docs/source/key_components/communication.md +++ b/docs/source/key_components/communication.md @@ -102,7 +102,7 @@ By registering the `conditional event` and related `handler function` to the register table, the handler function will be automatically executed with the received messages when the event conditions are met. -![Register Table](../images/distributed/register_table.svg) +![Register Table](../images/distributed/register_table.register.svg) `Conditional event` is used to declare the required message group for auto-triggering the related handler function. @@ -150,6 +150,8 @@ combined_event_mix = (("worker_01:update:2", "worker_02:update:3", "AND"), a specific conditional event. When the condition of the event is met, the related messages will be sent to the handler function for its execution. +![Register Table](../images/distributed/register_table.trigger.svg) + ```python # A common handler function signature def handler(that, proxy, messages): diff --git a/docs/source/key_components/rl_toolkit.md b/docs/source/key_components/rl_toolkit.md index 4c3b38d2..cb5ff795 100644 --- a/docs/source/key_components/rl_toolkit.md +++ b/docs/source/key_components/rl_toolkit.md @@ -69,8 +69,6 @@ the role of an agent assembler. It can assemble different RL agents according to the actual requirements, such as whether to share the underlying model, whether to share the experience pool, etc. -![Agent Manager](../images/rl/agent_manager.svg) - ```python def assemble_agents(self, config): # Initialize experience pool instance. @@ -87,6 +85,8 @@ Furthermore, to well serve the distributed algorithm (scalable), the agent manager provides two kinds of working modes, which can be applied in different distributed components, such as inference mode in actor, training mode in learner. +![Agent Manager](../images/rl/agent_manager.svg) + - In **inference mode**, the agent manager is responsible to access and shape the environment state for the related agent, convert the model action to an executable environment action, and finally generate experiences from the diff --git a/maro/simulator/scenarios/cim/common.py b/maro/simulator/scenarios/cim/common.py index 436fec23..23154e5e 100644 --- a/maro/simulator/scenarios/cim/common.py +++ b/maro/simulator/scenarios/cim/common.py @@ -100,8 +100,7 @@ class Action: return self.__str__() def __str__(self): - return f'Action {{quantity: {self.quantity}, port: {self.port_idx}, vessel: {self.vessel_idx} }}' - + return f'Action(port_idx={self.port_idx}, vessel_idx={self.vessel_idx}, quantity={self.quantity})' class ActionScope: """ @@ -115,8 +114,7 @@ class ActionScope: return self.__str__() def __str__(self): - return f'ActionScope {{load: {self.load}, discharge: {self.discharge} }}' - + return f'ActionScope(load={self.load}, discharge={self.discharge})' class DecisionEvent: """ diff --git a/maro/simulator/scenarios/citi_bike/common.py b/maro/simulator/scenarios/citi_bike/common.py index 2bf316ae..8928a510 100644 --- a/maro/simulator/scenarios/citi_bike/common.py +++ b/maro/simulator/scenarios/citi_bike/common.py @@ -51,7 +51,7 @@ class DecisionEvent: "action_scope": self.action_scope} def __repr__(self): - return f"decision event {self.__getstate__()}" + return self.__str__() def __str__(self): return f'DecisionEvent(tick={self.tick}, station_idx={self.station_idx}, type={self.type}, action_scope={self.action_scope})' @@ -63,6 +63,12 @@ class Action: self.to_station_idx = to_station_idx self.number = number + def __repr__(self): + return self.__str__() + + def __str__(self): + return f'Action(from_station_idx={self.from_station_idx}, to_station_idx={self.to_station_idx}, number={self.number})' + class ExtraCostMode(Enum): Source = "source" diff --git a/maro/simulator/scenarios/citi_bike/topologies/train/config.yml b/maro/simulator/scenarios/citi_bike/topologies/train/config.yml deleted file mode 100644 index 4dbdfb53..00000000 --- a/maro/simulator/scenarios/citi_bike/topologies/train/config.yml +++ /dev/null @@ -1,48 +0,0 @@ -decision: - extra_cost_mode: source # how to assign extra cost, avaiable value: source, target, target_neighbors - - resolution: 20 # frequency to check if a cell need an action - - # random factor to set bikes transfer time - effective_time_mean: 20 - effective_time_std: 10 - - # these 2 water mark will affect the if decision should be generated - supply_water_mark_ratio: 0.8 - demand_water_mark_ratio: 0.2 - - # ratio of action - action_scope: - #num: 20 # max avalible number of nerighbor station in each decision - low: 0.05 # min ratio of available bikes to keep for current cell, to supply to neighbors - high: 1 # max ratio of available bikes neighbors can provide to current cell - filters: # filters used to pick destinations - - type: "distance" # sort by distance, from neareast to farest - num: 40 # number of output - - type: "requirements" # sort by supply and demand requirements - num: 30 - - type: "trip_window" # sort by trips in window - windows: 10 # frame index number for the window - num: 20 - -reward: # reward options - fulfillment_factor: 0.4 - shortage_factor: 0.3 - transfer_cost_factor: 0.3 - -# timezone of the data -# NOTE: we need this if we want to fit local time, as binary data will convert timestamp into UTC -# name : https://en.wikipedia.org/wiki/List_of_tz_database_time_zones -time_zone: "America/New_York" - -# path to read trip data binary file -trip_data: "data/citibike/201901/trips.bin" - -# path to read weather data -weather_data: "data/citibike/KNYC_daily.bin" - -# path to read csv file that used to init stations, also with station id -> index mapping -stations_init_data: "data/citibike/201901/stations.csv" - -# path to distance adj matrix -distance_adj_data: "data/citibike/201901/distance_adj.csv" \ No newline at end of file diff --git a/maro/simulator/utils/common.py b/maro/simulator/utils/common.py index 20762fe1..9381bf0c 100644 --- a/maro/simulator/utils/common.py +++ b/maro/simulator/utils/common.py @@ -36,10 +36,12 @@ def get_scenarios()->List[str]: """ try: _, scenarios, _ = next(os.walk(scenarios_root_folder)) + scenarios = sorted([s for s in scenarios if not s.startswith("__")]) + except StopIteration: return [] - return [s for s in scenarios if not s.startswith("__")] + return scenarios def get_topologies(scenario: str)-> List[str]: """Get topology list of built-in scenario @@ -57,6 +59,8 @@ def get_topologies(scenario: str)-> List[str]: try: _, topologies, _ = next(os.walk(scenario_topology_root)) + topologies = sorted(topologies) + except StopIteration: return [] diff --git a/notebooks/bike_repositioning/interact_with_simulator.ipynb b/notebooks/bike_repositioning/interact_with_simulator.ipynb index 43531743..dc356115 100644 --- a/notebooks/bike_repositioning/interact_with_simulator.ipynb +++ b/notebooks/bike_repositioning/interact_with_simulator.ipynb @@ -18,13 +18,11 @@ "name": "stdout", "output_type": "stream", "text": [ - "04:14:36 | WARNING | \u001b[33mBinary data files for scenario: citi_bike topology: toy.3s_4t not found.\u001b[0m\n", - "04:14:36 | WARNING | \u001b[33mGenerating temp binary data file for scenario: citi_bike topology: toy.3s_4t pid: 119694. If you want to keep the data, please use MARO CLI command 'maro env data generate -s citi_bike -t toy.3s_4t' to generate the binary data files first.\u001b[0m\n", - "04:14:36 | INFO | \u001b[32mGenerating trip data for topology toy.3s_4t .\u001b[0m\n", - "04:14:37 | INFO | \u001b[32mCleaning weather data\u001b[0m\n", - "04:14:37 | INFO | \u001b[32mBuilding binary data from /home/Jinyu/.maro/data/citi_bike/.source/.clean/toy.3s_4t/87b553ccd7d54720/trips.csv to /home/Jinyu/.maro/data/citi_bike/.build/toy.3s_4t/87b553ccd7d54720/trips.bin\u001b[0m\n", - "04:14:46 | INFO | \u001b[32mBuilding binary data from /home/Jinyu/.maro/data/citi_bike/.source/.clean/toy.3s_4t/c0f13637d6834da0/weather.csv to /home/Jinyu/.maro/data/citi_bike/.build/toy.3s_4t/c0f13637d6834da0/KNYC_daily.bin\u001b[0m\n", - "{'perf': 0.4615742897065673, 'total_trips': 2147, 'total_shortage': 1156}\n" + "12:03:56 | WARNING | \u001b[33mBinary data files for scenario: citi_bike topology: toy.3s_4t not found.\u001b[0m\n", + "12:03:56 | WARNING | \u001b[33mGenerating temp binary data file for scenario: citi_bike topology: toy.3s_4t pid: 129013. If you want to keep the data, please use MARO CLI command 'maro env data generate -s citi_bike -t toy.3s_4t' to generate the binary data files first.\u001b[0m\n", + "12:03:56 | INFO | \u001b[32mGenerating trip data for topology toy.3s_4t .\u001b[0m\n", + "12:03:57 | INFO | \u001b[32mBuilding binary data from /home/Jinyu/.maro/data/citi_bike/.source/.clean/toy.3s_4t/b37f36f2de334eb3/trips.csv to /home/Jinyu/.maro/data/citi_bike/.build/toy.3s_4t/b37f36f2de334eb3/trips.bin\u001b[0m\n", + "{'trip_requirements': 2156, 'bike_shortage': 1163, 'operation_number': 0}\n" ] } ], @@ -78,40 +76,39 @@ "['cim', 'citi_bike']\n", "\n", "'The predefined topologies in Citi Bike:'\n", - "['ny.201912',\n", - " 'ny.201808',\n", - " 'ny.201907',\n", - " 'ny.202005',\n", - " 'ny.201812',\n", - " 'ny.201804',\n", - " 'toy.3s_4t',\n", - " 'toy.4s_4t',\n", - " 'ny.201908',\n", - " 'ny.201910',\n", - " 'train',\n", - " 'ny.201909',\n", - " 'ny.202002',\n", - " 'ny.201811',\n", - " 'ny.201906',\n", + "['ny.201801',\n", " 'ny.201802',\n", " 'ny.201803',\n", - " 'ny.201905',\n", - " 'ny.202003',\n", + " 'ny.201804',\n", " 'ny.201805',\n", - " 'ny.201809',\n", - " 'toy.5s_6t',\n", - " 'ny.201801',\n", - " 'ny.201904',\n", - " 'ny.201902',\n", - " 'ny.201901',\n", - " 'ny.201911',\n", - " 'ny.201903',\n", - " 'ny.202001',\n", - " 'ny.202004',\n", " 'ny.201806',\n", " 'ny.201807',\n", + " 'ny.201808',\n", + " 'ny.201809',\n", + " 'ny.201810',\n", + " 'ny.201811',\n", + " 'ny.201812',\n", + " 'ny.201901',\n", + " 'ny.201902',\n", + " 'ny.201903',\n", + " 'ny.201904',\n", + " 'ny.201905',\n", + " 'ny.201906',\n", + " 'ny.201907',\n", + " 'ny.201908',\n", + " 'ny.201909',\n", + " 'ny.201910',\n", + " 'ny.201911',\n", + " 'ny.201912',\n", + " 'ny.202001',\n", + " 'ny.202002',\n", + " 'ny.202003',\n", + " 'ny.202004',\n", + " 'ny.202005',\n", " 'ny.202006',\n", - " 'ny.201810']\n" + " 'toy.3s_4t',\n", + " 'toy.4s_4t',\n", + " 'toy.5s_6t']\n" ] } ], @@ -127,7 +124,7 @@ "pprint(scenarios)\n", "\n", "print()\n", - "pprint(f'The predefined topologies in Citi Bike:') # TODO: update the ordered output\n", + "pprint(f'The predefined topologies in Citi Bike:')\n", "pprint(topologies)" ] }, @@ -147,30 +144,50 @@ "name": "stdout", "output_type": "stream", "text": [ - "04:14:46 | WARNING | \u001b[33mBinary data files for scenario: citi_bike topology: toy.3s_4t not found.\u001b[0m\n", - "04:14:46 | WARNING | \u001b[33mGenerating temp binary data file for scenario: citi_bike topology: toy.3s_4t pid: 119694. If you want to keep the data, please use MARO CLI command 'maro env data generate -s citi_bike -t toy.3s_4t' to generate the binary data files first.\u001b[0m\n", - "04:14:46 | INFO | \u001b[32mGenerating trip data for topology toy.3s_4t .\u001b[0m\n", - "04:14:47 | INFO | \u001b[32mCleaning weather data\u001b[0m\n", - "04:14:47 | INFO | \u001b[32mBuilding binary data from /home/Jinyu/.maro/data/citi_bike/.source/.clean/toy.3s_4t/884f539bc7114e88/trips.csv to /home/Jinyu/.maro/data/citi_bike/.build/toy.3s_4t/884f539bc7114e88/trips.bin\u001b[0m\n", - "04:14:56 | INFO | \u001b[32mBuilding binary data from /home/Jinyu/.maro/data/citi_bike/.source/.clean/toy.3s_4t/d23d26ef5cdc45cb/weather.csv to /home/Jinyu/.maro/data/citi_bike/.build/toy.3s_4t/d23d26ef5cdc45cb/KNYC_daily.bin\u001b[0m\n", + "12:04:06 | WARNING | \u001b[33mBinary data files for scenario: citi_bike topology: toy.3s_4t not found.\u001b[0m\n", + "12:04:06 | WARNING | \u001b[33mGenerating temp binary data file for scenario: citi_bike topology: toy.3s_4t pid: 129013. If you want to keep the data, please use MARO CLI command 'maro env data generate -s citi_bike -t toy.3s_4t' to generate the binary data files first.\u001b[0m\n", + "12:04:06 | INFO | \u001b[32mGenerating trip data for topology toy.3s_4t .\u001b[0m\n", + "12:04:07 | INFO | \u001b[32mBuilding binary data from /home/Jinyu/.maro/data/citi_bike/.source/.clean/toy.3s_4t/0ad85988e84a43bd/trips.csv to /home/Jinyu/.maro/data/citi_bike/.build/toy.3s_4t/0ad85988e84a43bd/trips.bin\u001b[0m\n", "The current tick: 0.\n", "The current frame index: 0.\n", "There are 3 agents in this Env.\n", - "There will be 48 snapshots in total.\n" - ] - }, - { - "ename": "AttributeError", - "evalue": "'Station' object has no attribute 'id'", - "output_type": "error", - "traceback": [ - "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", - "\u001b[0;31mAttributeError\u001b[0m Traceback (most recent call last)", - "\u001b[0;32m\u001b[0m in \u001b[0;36m\u001b[0;34m\u001b[0m\n\u001b[1;32m 26\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 27\u001b[0m \u001b[0;31m# The summary info of the environment\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 28\u001b[0;31m \u001b[0msummary\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mdict\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0menv\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msummary\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 29\u001b[0m \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34mf\"\\nEnv Summary:\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 30\u001b[0m \u001b[0mpprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msummary\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m~/maro/maro/simulator/core.py\u001b[0m in \u001b[0;36msummary\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 115\u001b[0m \"\"\"\n\u001b[1;32m 116\u001b[0m return {\n\u001b[0;32m--> 117\u001b[0;31m \u001b[0;34m\"node_mapping\"\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_business_engine\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_node_mapping\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 118\u001b[0m \u001b[0;34m\"node_detail\"\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcurrent_frame\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_node_info\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 119\u001b[0m }\n", - "\u001b[0;32m~/maro/maro/simulator/scenarios/citi_bike/business_engine.py\u001b[0m in \u001b[0;36mget_node_mapping\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 138\u001b[0m \u001b[0mnode_mapping\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m{\u001b[0m\u001b[0;34m}\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 139\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mstation\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_stations\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 140\u001b[0;31m \u001b[0mnode_mapping\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mstation\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mindex\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mstation\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mid\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 141\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0mnode_mapping\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 142\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m~/maro/maro/backends/frame.pyx\u001b[0m in \u001b[0;36mmaro.backends.frame.NodeBase.__getattribute__\u001b[0;34m()\u001b[0m\n", - "\u001b[0;31mAttributeError\u001b[0m: 'Station' object has no attribute 'id'" + "There will be 48 snapshots in total.\n", + "\n", + "Env Summary:\n", + "{'node_detail': {'matrices': {'attributes': {'trips_adj': {'slots': 9,\n", + " 'type': 'i'}},\n", + " 'number': 1},\n", + " 'stations': {'attributes': {'bikes': {'slots': 1, 'type': 'i'},\n", + " 'capacity': {'slots': 1,\n", + " 'type': 'i'},\n", + " 'extra_cost': {'slots': 1,\n", + " 'type': 'i'},\n", + " 'failed_return': {'slots': 1,\n", + " 'type': 'i'},\n", + " 'fulfillment': {'slots': 1,\n", + " 'type': 'i'},\n", + " 'holiday': {'slots': 1,\n", + " 'type': 'i2'},\n", + " 'id': {'slots': 1, 'type': 'i'},\n", + " 'min_bikes': {'slots': 1,\n", + " 'type': 'i'},\n", + " 'shortage': {'slots': 1,\n", + " 'type': 'i'},\n", + " 'temperature': {'slots': 1,\n", + " 'type': 'i2'},\n", + " 'transfer_cost': {'slots': 1,\n", + " 'type': 'i'},\n", + " 'trip_requirement': {'slots': 1,\n", + " 'type': 'i'},\n", + " 'weather': {'slots': 1,\n", + " 'type': 'i2'},\n", + " 'weekday': {'slots': 1,\n", + " 'type': 'i2'}},\n", + " 'number': 3}},\n", + " 'node_mapping': {0: 0, 1: 1, 2: 2}}\n", + "\n", + "Env Metrics:\n", + "{'trip_requirements': 0, 'bike_shortage': 0, 'operation_number': 0}\n" ] } ], @@ -208,7 +225,7 @@ "\n", "# The metrics of the environment\n", "metrics: dict = env.metrics\n", - "print(f\"\\nEnv Metrics:\") # TODO: update the output with node mapping\n", + "print(f\"\\nEnv Metrics:\")\n", "pprint(metrics)" ] }, @@ -258,30 +275,19 @@ "name": "stdout", "output_type": "stream", "text": [ - "04:16:02 | WARNING | \u001b[33mBinary data files for scenario: citi_bike topology: toy.3s_4t not found.\u001b[0m\n", - "04:16:02 | WARNING | \u001b[33mGenerating temp binary data file for scenario: citi_bike topology: toy.3s_4t pid: 119694. If you want to keep the data, please use MARO CLI command 'maro env data generate -s citi_bike -t toy.3s_4t' to generate the binary data files first.\u001b[0m\n", - "04:16:02 | INFO | \u001b[32mGenerating trip data for topology toy.3s_4t .\u001b[0m\n", - "04:16:03 | INFO | \u001b[32mCleaning weather data\u001b[0m\n", - "04:16:03 | INFO | \u001b[32mBuilding binary data from /home/Jinyu/.maro/data/citi_bike/.source/.clean/toy.3s_4t/067f8d887e094457/trips.csv to /home/Jinyu/.maro/data/citi_bike/.build/toy.3s_4t/067f8d887e094457/trips.bin\u001b[0m\n", - "04:16:11 | INFO | \u001b[32mBuilding binary data from /home/Jinyu/.maro/data/citi_bike/.source/.clean/toy.3s_4t/7863dcc1cc5440ba/weather.csv to /home/Jinyu/.maro/data/citi_bike/.build/toy.3s_4t/7863dcc1cc5440ba/KNYC_daily.bin\u001b[0m\n", + "12:04:16 | WARNING | \u001b[33mBinary data files for scenario: citi_bike topology: toy.3s_4t not found.\u001b[0m\n", + "12:04:16 | WARNING | \u001b[33mGenerating temp binary data file for scenario: citi_bike topology: toy.3s_4t pid: 129013. If you want to keep the data, please use MARO CLI command 'maro env data generate -s citi_bike -t toy.3s_4t' to generate the binary data files first.\u001b[0m\n", + "12:04:16 | INFO | \u001b[32mGenerating trip data for topology toy.3s_4t .\u001b[0m\n", + "12:04:17 | INFO | \u001b[32mBuilding binary data from /home/Jinyu/.maro/data/citi_bike/.source/.clean/toy.3s_4t/56e2ca55ba6d4881/trips.csv to /home/Jinyu/.maro/data/citi_bike/.build/toy.3s_4t/56e2ca55ba6d4881/trips.bin\u001b[0m\n", "*************\n", - "DecisionEvent(tick=479, station_idx=0, type=DecisionType.Demand, action_scope={2: 0, 1: 1, 0: 29})\n", - "\n", + "DecisionEvent(tick=79, station_idx=1, type=DecisionType.Demand, action_scope={0: 0, 2: 0, 1: 30})\n", + "Action(from_station_idx=2, to_station_idx=1, number=0)\n", "*************\n", - "DecisionEvent(tick=539, station_idx=2, type=DecisionType.Demand, action_scope={0: 0, 1: 0, 2: 29})\n", - "\n", + "DecisionEvent(tick=799, station_idx=2, type=DecisionType.Demand, action_scope={0: 0, 1: 0, 2: 30})\n", + "Action(from_station_idx=1, to_station_idx=2, number=0)\n", "*************\n", - "DecisionEvent(tick=619, station_idx=0, type=DecisionType.Demand, action_scope={2: 1, 1: 3, 0: 30})\n", - "\n", - "*************\n", - "DecisionEvent(tick=959, station_idx=2, type=DecisionType.Demand, action_scope={0: 0, 1: 0, 2: 29})\n", - "\n", - "*************\n", - "DecisionEvent(tick=1059, station_idx=2, type=DecisionType.Demand, action_scope={1: 0, 0: 0, 2: 30})\n", - "\n", - "*************\n", - "DecisionEvent(tick=1319, station_idx=0, type=DecisionType.Demand, action_scope={2: 1, 1: 0, 0: 30})\n", - "\n" + "DecisionEvent(tick=959, station_idx=2, type=DecisionType.Demand, action_scope={0: 1, 1: 1, 2: 30})\n", + "Action(from_station_idx=0, to_station_idx=2, number=0)\n" ] } ], @@ -345,8 +351,8 @@ " action = None\n", " \n", " # Randomly sample some records to show in the output\n", - " if random.random() > 0.95:\n", - " print(f\"*************\\n{decision_event}\\n{action}\") # TODO: update the output\n", + " if random.random() > 0.98:\n", + " print(f\"*************\\n{decision_event}\\n{action}\")\n", " \n", " # Respond the environment with the generated Action\n", " metric, decision_event, is_done = env.step(action)" @@ -367,7 +373,7 @@ "\n", "The return value from the snapshot list is a numpy.ndarray with shape **(num_frame * num_station * num_attribute, )**.\n", "\n", - "More detailed introduction to the snapshot list is [here](). # TODO: add hyper-link" + "More detailed introduction to the snapshot list is [here](https://maro.readthedocs.io/en/latest/key_components/data_model.html#advanced-features)." ] }, { @@ -379,26 +385,28 @@ "name": "stdout", "output_type": "stream", "text": [ - "04:18:04 | WARNING | \u001b[33mBinary data files for scenario: citi_bike topology: toy.3s_4t not found.\u001b[0m\n", - "04:18:04 | WARNING | \u001b[33mGenerating temp binary data file for scenario: citi_bike topology: toy.3s_4t pid: 119694. If you want to keep the data, please use MARO CLI command 'maro env data generate -s citi_bike -t toy.3s_4t' to generate the binary data files first.\u001b[0m\n", - "04:18:04 | INFO | \u001b[32mGenerating trip data for topology toy.3s_4t .\u001b[0m\n", - "04:18:05 | INFO | \u001b[32mCleaning weather data\u001b[0m\n", - "04:18:05 | INFO | \u001b[32mBuilding binary data from /home/Jinyu/.maro/data/citi_bike/.source/.clean/toy.3s_4t/6ca7bfe80c304dc5/trips.csv to /home/Jinyu/.maro/data/citi_bike/.build/toy.3s_4t/6ca7bfe80c304dc5/trips.bin\u001b[0m\n", - "04:18:13 | INFO | \u001b[32mBuilding binary data from /home/Jinyu/.maro/data/citi_bike/.source/.clean/toy.3s_4t/c1505f6fe88e4d96/weather.csv to /home/Jinyu/.maro/data/citi_bike/.build/toy.3s_4t/c1505f6fe88e4d96/KNYC_daily.bin\u001b[0m\n" - ] - }, - { - "ename": "AttributeError", - "evalue": "'Station' object has no attribute 'id'", - "output_type": "error", - "traceback": [ - "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", - "\u001b[0;31mAttributeError\u001b[0m Traceback (most recent call last)", - "\u001b[0;32m\u001b[0m in \u001b[0;36m\u001b[0;34m\u001b[0m\n\u001b[1;32m 7\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 8\u001b[0m \u001b[0;31m# To get the attribute list that can be accessed in snapshot_list\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 9\u001b[0;31m \u001b[0mpprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0menv\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msummary\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'node_detail'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdepth\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m2\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 10\u001b[0m \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 11\u001b[0m \u001b[0;31m# The attribute list of stations\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m~/maro/maro/simulator/core.py\u001b[0m in \u001b[0;36msummary\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 115\u001b[0m \"\"\"\n\u001b[1;32m 116\u001b[0m return {\n\u001b[0;32m--> 117\u001b[0;31m \u001b[0;34m\"node_mapping\"\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_business_engine\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_node_mapping\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 118\u001b[0m \u001b[0;34m\"node_detail\"\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcurrent_frame\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_node_info\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 119\u001b[0m }\n", - "\u001b[0;32m~/maro/maro/simulator/scenarios/citi_bike/business_engine.py\u001b[0m in \u001b[0;36mget_node_mapping\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 138\u001b[0m \u001b[0mnode_mapping\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m{\u001b[0m\u001b[0;34m}\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 139\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mstation\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_stations\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 140\u001b[0;31m \u001b[0mnode_mapping\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mstation\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mindex\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mstation\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mid\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 141\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0mnode_mapping\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 142\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m~/maro/maro/backends/frame.pyx\u001b[0m in \u001b[0;36mmaro.backends.frame.NodeBase.__getattribute__\u001b[0;34m()\u001b[0m\n", - "\u001b[0;31mAttributeError\u001b[0m: 'Station' object has no attribute 'id'" + "12:04:26 | WARNING | \u001b[33mBinary data files for scenario: citi_bike topology: toy.3s_4t not found.\u001b[0m\n", + "12:04:26 | WARNING | \u001b[33mGenerating temp binary data file for scenario: citi_bike topology: toy.3s_4t pid: 129013. If you want to keep the data, please use MARO CLI command 'maro env data generate -s citi_bike -t toy.3s_4t' to generate the binary data files first.\u001b[0m\n", + "12:04:26 | INFO | \u001b[32mGenerating trip data for topology toy.3s_4t .\u001b[0m\n", + "12:04:27 | INFO | \u001b[32mBuilding binary data from /home/Jinyu/.maro/data/citi_bike/.source/.clean/toy.3s_4t/278c7458875b4474/trips.csv to /home/Jinyu/.maro/data/citi_bike/.build/toy.3s_4t/278c7458875b4474/trips.bin\u001b[0m\n", + "{'matrices': {'attributes': {...}, 'number': 1},\n", + " 'stations': {'attributes': {...}, 'number': 3}}\n", + "\n", + "{'attributes': {'bikes': {'slots': 1, 'type': 'i'},\n", + " 'capacity': {'slots': 1, 'type': 'i'},\n", + " 'extra_cost': {'slots': 1, 'type': 'i'},\n", + " 'failed_return': {'slots': 1, 'type': 'i'},\n", + " 'fulfillment': {'slots': 1, 'type': 'i'},\n", + " 'holiday': {'slots': 1, 'type': 'i2'},\n", + " 'id': {'slots': 1, 'type': 'i'},\n", + " 'min_bikes': {'slots': 1, 'type': 'i'},\n", + " 'shortage': {'slots': 1, 'type': 'i'},\n", + " 'temperature': {'slots': 1, 'type': 'i2'},\n", + " 'transfer_cost': {'slots': 1, 'type': 'i'},\n", + " 'trip_requirement': {'slots': 1, 'type': 'i'},\n", + " 'weather': {'slots': 1, 'type': 'i2'},\n", + " 'weekday': {'slots': 1, 'type': 'i2'}},\n", + " 'number': 3}\n" ] } ], @@ -421,7 +429,20 @@ "cell_type": "code", "execution_count": null, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "12:04:35 | WARNING | \u001b[33mBinary data files for scenario: citi_bike topology: toy.3s_4t not found.\u001b[0m\n", + "12:04:35 | WARNING | \u001b[33mGenerating temp binary data file for scenario: citi_bike topology: toy.3s_4t pid: 129013. If you want to keep the data, please use MARO CLI command 'maro env data generate -s citi_bike -t toy.3s_4t' to generate the binary data files first.\u001b[0m\n", + "12:04:35 | INFO | \u001b[32mGenerating trip data for topology toy.3s_4t .\u001b[0m\n", + "12:04:36 | INFO | \u001b[32mBuilding binary data from /home/Jinyu/.maro/data/citi_bike/.source/.clean/toy.3s_4t/80aa59d9c58d4b5e/trips.csv to /home/Jinyu/.maro/data/citi_bike/.build/toy.3s_4t/80aa59d9c58d4b5e/trips.bin\u001b[0m\n", + "array([11., 0., 11., 15., 0., 15., 16., 0., 16., 16., 0., 16.],\n", + " dtype=float32)\n" + ] + } + ], "source": [ "from maro.backends.frame import SnapshotList\n", "from maro.simulator import Env\n", diff --git a/notebooks/container_inventory_management/implement_rl_agent.ipynb b/notebooks/container_inventory_management/implement_rl_agent.ipynb index ed75d806..c0f1f238 100644 --- a/notebooks/container_inventory_management/implement_rl_agent.ipynb +++ b/notebooks/container_inventory_management/implement_rl_agent.ipynb @@ -1,11 +1,304 @@ { "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# State Shaper\n", + "\n", + "A state shaper is used to convert an environment observation to a state vector as input to value or policy models by extracting relevant temporal and spatial information. The scenario-specific call method returns the the ID of the agent involved in the current decision event and the shaped state." + ] + }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], - "source": [] + "source": [ + "import numpy as np\n", + "from maro.rl import StateShaper\n", + "\n", + "\n", + "class CIMStateShaper(StateShaper):\n", + " def __init__(self, *, look_back, max_ports_downstream, port_attributes, vessel_attributes):\n", + " super().__init__()\n", + " self._look_back = look_back\n", + " self._max_ports_downstream = max_ports_downstream\n", + " self._port_attributes = port_attributes\n", + " self._vessel_attributes = vessel_attributes\n", + " self._dim = (look_back + 1) * (max_ports_downstream + 1) * len(port_attributes) + len(vessel_attributes)\n", + "\n", + " def __call__(self, decision_event, snapshot_list):\n", + " tick, port_idx, vessel_idx = decision_event.tick, decision_event.port_idx, decision_event.vessel_idx\n", + " ticks = [tick - rt for rt in range(self._look_back-1)]\n", + " future_port_idx_list = snapshot_list[\"vessels\"][tick: vessel_idx: 'future_stop_list'].astype('int')\n", + " port_features = snapshot_list[\"ports\"][ticks: [port_idx] + list(future_port_idx_list): self._port_attributes]\n", + " vessel_features = snapshot_list[\"vessels\"][tick: vessel_idx: self._vessel_attributes]\n", + " state = np.concatenate((port_features, vessel_features))\n", + " return str(port_idx), state\n", + " \n", + " @property\n", + " def dim(self):\n", + " return self._dim" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Action Shaper\n", + "\n", + "An action shaper is used to convert an agent's output to an Action object which can be executed by the env's step() method." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from maro.rl import ActionShaper\n", + "from maro.simulator.scenarios.cim.common import Action\n", + "\n", + "\n", + "class CIMActionShaper(ActionShaper):\n", + " def __init__(self, action_space):\n", + " super().__init__()\n", + " self._action_space = action_space\n", + " self._zero_action_index = action_space.index(0)\n", + "\n", + " def __call__(self, model_action, decision_event, snapshot_list):\n", + " assert 0 <= model_action < len(self._action_space)\n", + " \n", + " scope = decision_event.action_scope\n", + " tick = decision_event.tick\n", + " port_idx = decision_event.port_idx\n", + " vessel_idx = decision_event.vessel_idx\n", + " port_empty = snapshot_list[\"ports\"][tick: port_idx: [\"empty\", \"full\", \"on_shipper\", \"on_consignee\"]][0]\n", + " vessel_remaining_space = snapshot_list[\"vessels\"][tick: vessel_idx: [\"empty\", \"full\", \"remaining_space\"]][2]\n", + " early_discharge = snapshot_list[\"vessels\"][tick:vessel_idx: \"early_discharge\"][0]\n", + "\n", + " if model_action < self._zero_action_index:\n", + " actual_action = max(round(self._action_space[model_action] * port_empty), -vessel_remaining_space)\n", + " elif model_action > self._zero_action_index:\n", + " plan_action = self._action_space[model_action] * (scope.discharge + early_discharge) - early_discharge\n", + " actual_action = round(plan_action) if plan_action > 0 else round(self._action_space[model_action] * scope.discharge)\n", + " else:\n", + " actual_action = 0\n", + "\n", + " return Action(vessel_idx, port_idx, actual_action)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Experience Shaper\n", + "\n", + "An experience shaper is responsible for converting the trajectory from an episode to experiences that can be used by the agents' training algorithms. It is necessary to specify how to determine the reward for an action given the business metrics associated with the corresponding transition. Here we showcase a custom reward shaper for the contained inventory management (CIM) scenario. " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from collections import defaultdict\n", + "\n", + "from maro.rl import ExperienceShaper\n", + "\n", + "\n", + "class TruncatedExperienceShaper(ExperienceShaper):\n", + " def __init__(self, *, time_window: int, time_decay_factor: float, fulfillment_factor: float,\n", + " shortage_factor: float):\n", + " super().__init__(reward_func=None)\n", + " self._time_window = time_window\n", + " self._time_decay_factor = time_decay_factor\n", + " self._fulfillment_factor = fulfillment_factor\n", + " self._shortage_factor = shortage_factor\n", + "\n", + " def __call__(self, trajectory, snapshot_list):\n", + " experiences_by_agent = {}\n", + " for i in range(len(trajectory) - 1):\n", + " transition = trajectory[i]\n", + " agent_id = transition[\"agent_id\"]\n", + " if agent_id not in experiences_by_agent:\n", + " experiences_by_agent[agent_id] = defaultdict(list)\n", + " \n", + " experiences = experiences_by_agent[agent_id]\n", + " experiences[\"state\"].append(transition[\"state\"])\n", + " experiences[\"action\"].append(transition[\"action\"])\n", + " experiences[\"reward\"].append(self._compute_reward(transition[\"event\"], snapshot_list))\n", + " experiences[\"next_state\"].append(trajectory[i+1][\"state\"])\n", + "\n", + " return experiences_by_agent\n", + "\n", + " def _compute_reward(self, decision_event, snapshot_list):\n", + " start_tick = decision_event.tick + 1\n", + " end_tick = decision_event.tick + self._time_window\n", + " ticks = list(range(start_tick, end_tick))\n", + "\n", + " # calculate tc reward\n", + " future_fulfillment = snapshot_list[\"ports\"][ticks::\"fulfillment\"]\n", + " future_shortage = snapshot_list[\"ports\"][ticks::\"shortage\"]\n", + " decay_list = [self._time_decay_factor ** i for i in range(end_tick - start_tick)\n", + " for _ in range(future_fulfillment.shape[0]//(end_tick-start_tick))]\n", + "\n", + " tot_fulfillment = np.dot(future_fulfillment, decay_list)\n", + " tot_shortage = np.dot(future_shortage, decay_list)\n", + "\n", + " return np.float(self._fulfillment_factor * tot_fulfillment - self._shortage_factor * tot_shortage)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Agent\n", + "\n", + "An agent is a combination of (RL) algorithm, experience pool, and a set of non-algorithm-specific parameters (algorithm-specific parameters are managed by the algorithm module). Non-algorithm-specific parameters are used to manage experience storage, sampling strategies, and training strategies. Since all kinds of scenario-specific stuff will be handled by the agent manager, the agent is scenario agnostic." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from maro.rl import AbsAgent, ColumnBasedStore\n", + "\n", + "\n", + "class CIMAgent(AbsAgent):\n", + " def __init__(self, name, algorithm, experience_pool: ColumnBasedStore, min_experiences_to_train,\n", + " num_batches, batch_size):\n", + " super().__init__(name, algorithm, experience_pool)\n", + " self._min_experiences_to_train = min_experiences_to_train\n", + " self._num_batches = num_batches\n", + " self._batch_size = batch_size\n", + "\n", + " def train(self):\n", + " if len(self._experience_pool) < self._min_experiences_to_train:\n", + " return\n", + "\n", + " for _ in range(self._num_batches):\n", + " indexes, sample = self._experience_pool.sample_by_key(\"loss\", self._batch_size)\n", + " state = np.asarray(sample[\"state\"])\n", + " action = np.asarray(sample[\"action\"])\n", + " reward = np.asarray(sample[\"reward\"])\n", + " next_state = np.asarray(sample[\"next_state\"])\n", + " loss = self._algorithm.train(state, action, reward, next_state)\n", + " self._experience_pool.update(indexes, {\"loss\": loss})" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Agent Manager\n", + "\n", + "An agent manager manages all agents and provides a unified interface with the environment. The agent manager is responsible for both inference and training. It is composed of a state shaper, an action shaper and an experience shaper which perform necessary conversions so that the underlying agents do not need to concern themselves with the business logic. " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import io\n", + "import yaml\n", + "\n", + "from torch.nn.functional import smooth_l1_loss\n", + "from torch.optim import RMSprop\n", + "\n", + "from maro.rl import AbsAgentManager, LearningModel, MLPDecisionLayers, DQN, DQNHyperParams, ColumnBasedStore\n", + "\n", + "\n", + "num_actions = 21\n", + "\n", + "\n", + "class DQNAgentManager(AbsAgentManager):\n", + " def _assemble(self, agent_dict):\n", + " for agent_id in self._agent_id_list:\n", + " eval_model = LearningModel(decision_layers=MLPDecisionLayers(name=f'{agent_id}.policy',\n", + " input_dim=self._state_shaper.dim,\n", + " output_dim=num_actions,\n", + " hidden_dims=[256, 128, 64],\n", + " dropout_p=.0)\n", + " )\n", + "\n", + " algorithm = DQN(model_dict={\"eval\": eval_model},\n", + " optimizer_opt=(RMSprop, {\"lr\": 0.05}),\n", + " loss_func_dict={\"eval\": smooth_l1_loss},\n", + " hyper_params=DQNHyperParams(num_actions=num_actions, reward_decay=.0,\n", + " num_training_rounds_per_target_replacement=5, tau=0.1)\n", + " )\n", + "\n", + " experience_pool = ColumnBasedStore()\n", + " \n", + " agent_dict[agent_id] = CIMAgent(name=agent_id, algorithm=algorithm, experience_pool=experience_pool,\n", + " min_experiences_to_train=1024, num_batches=10, batch_size=128)\n", + "\n", + " def store_experiences(self, experiences):\n", + " for agent_id, exp in experiences.items():\n", + " exp.update({\"loss\": [1e8] * len(exp[next(iter(exp))])})\n", + " self._agent_dict[agent_id].store_experiences(exp)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Main Loop\n", + "\n", + "The code below demonstrates the typical structure of a program using MARO. One starts by creating an environment. Next, shapers and an explorer are created and an agent manager is created that loads these components. The creation of the agent manager also assembles all agents under the hood. Because the code is for the single-host mode, the agent manager mode is set to TRAIN_INFERENCE. Next, an actor is created to wrap the env and agent manager, and a learner is created to wrap the same agent manager and the actor. Finally, the task is started by calling the learner's train() method." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from maro.simulator import Env\n", + "from maro.rl import SimpleLearner, SimpleActor, AgentMode, TwoPhaseLinearExplorer\n", + "from maro.utils import Logger\n", + "\n", + "\n", + "env = Env(\"cim\", \"toy.4p_ssdd_l0.0\", durations=1120)\n", + "total_episodes = 100\n", + "agent_id_list = [str(agent_id) for agent_id in env.agent_idx_list]\n", + "\n", + "state_shaper = CIMStateShaper(look_back=7, max_ports_downstream=2, \n", + " port_attributes=[\"empty\", \"full\", \"on_shipper\", \"on_consignee\", \n", + " \"booking\", \"shortage\", \"fulfillment\"],\n", + " vessel_attributes=[\"empty\", \"full\", \"remaining_space\"]\n", + " )\n", + "\n", + "action_shaper = CIMActionShaper(action_space=list(np.linspace(-1.0, 1.0, num_actions)))\n", + "\n", + "experience_shaper = TruncatedExperienceShaper(time_window=100, fulfillment_factor=1.0, shortage_factor=1.0,\n", + " time_decay_factor=0.97)\n", + "\n", + "explorer = TwoPhaseLinearExplorer(agent_id_list, total_episodes, \n", + " epsilon_range_dict={\"_all_\": (.0, .4)},\n", + " split_point_dict={\"_all_\": (.5, .8)},\n", + " with_cache=True)\n", + "\n", + "agent_manager = DQNAgentManager(name=\"cim_learner\",\n", + " mode=AgentMode.TRAIN_INFERENCE,\n", + " agent_id_list=agent_id_list,\n", + " state_shaper=state_shaper,\n", + " action_shaper=action_shaper,\n", + " experience_shaper=experience_shaper,\n", + " explorer=explorer)\n", + "\n", + "learner = SimpleLearner(trainable_agents=agent_manager, actor=SimpleActor(env, agent_manager),\n", + " logger=Logger(\"single_host_cim_learner\", auto_timestamp=False))\n", + "\n", + "learner.train(total_episodes)" + ] } ], "metadata": { diff --git a/notebooks/container_inventory_management/interact_with_simulator.ipynb b/notebooks/container_inventory_management/interact_with_simulator.ipynb index 590465f6..6cf38243 100644 --- a/notebooks/container_inventory_management/interact_with_simulator.ipynb +++ b/notebooks/container_inventory_management/interact_with_simulator.ipynb @@ -18,7 +18,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "{'perf': 0.5, 'total_shortage': 100000, 'total_cost': 0}\n" + "{'order_requirements': 200000, 'container_shortage': 100000, 'operation_number': 0}\n" ] } ], @@ -70,42 +70,42 @@ "['cim', 'citi_bike']\n", "\n", "'The predefined topologies in CIM:'\n", - "['toy.6p_sssbdd_l0.3',\n", - " 'toy.4p_ssdd_l0.2',\n", - " 'global_trade.22p_l0.7',\n", - " 'toy.5p_ssddd_l0.1',\n", - " 'toy.6p_sssbdd_l0.1',\n", - " 'toy.5p_ssddd_l0.8',\n", - " 'toy.6p_sssbdd_l0.8',\n", - " 'toy.6p_sssbdd_l0.2',\n", - " 'toy.6p_sssbdd_l0.5',\n", - " 'toy.6p_sssbdd_l0.0',\n", - " 'toy.6p_sssbdd_l0.7',\n", - " 'toy.5p_ssddd_l0.3',\n", - " 'toy.4p_ssdd_l0.3',\n", - " 'toy.5p_ssddd_l0.5',\n", - " 'toy.4p_ssdd_l0.8',\n", - " 'toy.4p_ssdd_l0.1',\n", - " 'toy.6p_sssbdd_l0.4',\n", - " 'toy.5p_ssddd_l0.7',\n", - " 'global_trade.22p_l0.3',\n", - " 'global_trade.22p_l0.8',\n", - " 'global_trade.22p_l0.2',\n", - " 'toy.4p_ssdd_l0.4',\n", - " 'toy.6p_sssbdd_l0.6',\n", - " 'toy.4p_ssdd_l0.7',\n", - " 'toy.5p_ssddd_l0.4',\n", + "['global_trade.22p_l0.0',\n", " 'global_trade.22p_l0.1',\n", - " 'global_trade.22p_l0.0',\n", - " 'global_trade.22p_l0.6',\n", - " 'toy.4p_ssdd_l0.5',\n", - " 'toy.5p_ssddd_l0.2',\n", - " 'toy.4p_ssdd_l0.0',\n", - " 'global_trade.22p_l0.5',\n", - " 'toy.5p_ssddd_l0.0',\n", - " 'toy.5p_ssddd_l0.6',\n", + " 'global_trade.22p_l0.2',\n", + " 'global_trade.22p_l0.3',\n", " 'global_trade.22p_l0.4',\n", - " 'toy.4p_ssdd_l0.6']\n" + " 'global_trade.22p_l0.5',\n", + " 'global_trade.22p_l0.6',\n", + " 'global_trade.22p_l0.7',\n", + " 'global_trade.22p_l0.8',\n", + " 'toy.4p_ssdd_l0.0',\n", + " 'toy.4p_ssdd_l0.1',\n", + " 'toy.4p_ssdd_l0.2',\n", + " 'toy.4p_ssdd_l0.3',\n", + " 'toy.4p_ssdd_l0.4',\n", + " 'toy.4p_ssdd_l0.5',\n", + " 'toy.4p_ssdd_l0.6',\n", + " 'toy.4p_ssdd_l0.7',\n", + " 'toy.4p_ssdd_l0.8',\n", + " 'toy.5p_ssddd_l0.0',\n", + " 'toy.5p_ssddd_l0.1',\n", + " 'toy.5p_ssddd_l0.2',\n", + " 'toy.5p_ssddd_l0.3',\n", + " 'toy.5p_ssddd_l0.4',\n", + " 'toy.5p_ssddd_l0.5',\n", + " 'toy.5p_ssddd_l0.6',\n", + " 'toy.5p_ssddd_l0.7',\n", + " 'toy.5p_ssddd_l0.8',\n", + " 'toy.6p_sssbdd_l0.0',\n", + " 'toy.6p_sssbdd_l0.1',\n", + " 'toy.6p_sssbdd_l0.2',\n", + " 'toy.6p_sssbdd_l0.3',\n", + " 'toy.6p_sssbdd_l0.4',\n", + " 'toy.6p_sssbdd_l0.5',\n", + " 'toy.6p_sssbdd_l0.6',\n", + " 'toy.6p_sssbdd_l0.7',\n", + " 'toy.6p_sssbdd_l0.8']\n" ] } ], @@ -121,7 +121,7 @@ "pprint(scenarios)\n", "\n", "print()\n", - "pprint(f'The predefined topologies in CIM:') # TODO: update the ordered output\n", + "pprint(f'The predefined topologies in CIM:')\n", "pprint(topologies)" ] }, @@ -210,7 +210,7 @@ " 'rt2_vessel_003': 5}}}\n", "\n", "Env Metrics:\n", - "{'perf': 1, 'total_shortage': 0, 'total_cost': 0}\n" + "{'order_requirements': 0, 'container_shortage': 0, 'operation_number': 0}\n" ] } ], @@ -298,14 +298,11 @@ "output_type": "stream", "text": [ "*************\n", - "DecisionEvent(tick=35, port_idx=3, vessel_idx=0, action_scope=ActionScope {load: 17788, discharge: 15403 })\n", - "Action {quantity: -14533, port: 3, vessel: 0 }\n", + "DecisionEvent(tick=14, port_idx=0, vessel_idx=5, action_scope=ActionScope(load=8856, discharge=1981))\n", + "Action(port_idx=0, vessel_idx=5, quantity=-2667)\n", "*************\n", - "DecisionEvent(tick=56, port_idx=1, vessel_idx=3, action_scope=ActionScope {load: 0, discharge: 5 })\n", - "Action {quantity: 4, port: 1, vessel: 3 }\n", - "*************\n", - "DecisionEvent(tick=98, port_idx=4, vessel_idx=4, action_scope=ActionScope {load: 2047, discharge: 85 })\n", - "Action {quantity: -1818, port: 4, vessel: 4 }\n" + "DecisionEvent(tick=21, port_idx=2, vessel_idx=1, action_scope=ActionScope(load=15997, discharge=3061))\n", + "Action(port_idx=2, vessel_idx=1, quantity=-11608)\n" ] } ], @@ -361,7 +358,7 @@ "\n", "The return value from the snapshot list is a numpy.ndarray with shape **(num_tick * num_port * num_attribute, )**.\n", "\n", - "More detailed introduction to the snapshot list is [here](docs/_build/html/key_components/data_model.html#advanced-features). # TODO: add hyper-link" + "More detailed introduction to the snapshot list is [here](https://maro.readthedocs.io/en/latest/key_components/data_model.html#advanced-features)." ] }, {