From b06849a04642e86f3126e51883971a2b82c21818 Mon Sep 17 00:00:00 2001 From: Ross Gardler Date: Mon, 12 Mar 2018 16:21:49 +0000 Subject: [PATCH] add back documentation and test files from SimDem1 --- demo_scripts/README.md | 19 ++ demo_scripts/env.json | 3 + demo_scripts/simdem/README.md | 48 +++++ demo_scripts/simdem/building/README.md | 13 ++ demo_scripts/simdem/demo/README.md | 74 +++++++ demo_scripts/simdem/env.json | 3 + demo_scripts/simdem/env.local.json | 3 + demo_scripts/simdem/modes/README.md | 95 +++++++++ demo_scripts/simdem/multipart/README.md | 106 ++++++++++ demo_scripts/simdem/prerequisites/README.md | 96 +++++++++ demo_scripts/simdem/prerequisites/remote.md | 4 + demo_scripts/simdem/running/README.md | 56 +++++ .../simdem/special_commands/README.md | 37 ++++ demo_scripts/simdem/syntax/README.md | 120 +++++++++++ demo_scripts/simdem/test/README.md | 89 ++++++++ demo_scripts/simdem/tutorial/README.md | 46 +++++ demo_scripts/simdem/variables/README.md | 168 +++++++++++++++ demo_scripts/simdem/variables/env.json | 3 + demo_scripts/simdem/variables/env.local.json | 3 + demo_scripts/test/README.md | 157 ++++++++++++++ demo_scripts/test/directory/README.md | 26 +++ demo_scripts/test/env.json | 4 + demo_scripts/test/env.local.json | 3 + demo_scripts/test/env.test.json | 3 + demo_scripts/test/environment_test.md | 195 ++++++++++++++++++ demo_scripts/test/prerequisites/README.md | 33 +++ .../test/prerequisites/nested_prereq.md | 25 +++ .../prerequisites/passing_prerequisite.md | 28 +++ demo_scripts/test/remote/README.md | 33 +++ demo_scripts/test/test_plan.txt | 4 + 30 files changed, 1497 insertions(+) create mode 100644 demo_scripts/README.md create mode 100644 demo_scripts/env.json create mode 100644 demo_scripts/simdem/README.md create mode 100644 demo_scripts/simdem/building/README.md create mode 100644 demo_scripts/simdem/demo/README.md create mode 100644 demo_scripts/simdem/env.json create mode 100644 demo_scripts/simdem/env.local.json create mode 100644 demo_scripts/simdem/modes/README.md create mode 100644 demo_scripts/simdem/multipart/README.md create mode 100644 demo_scripts/simdem/prerequisites/README.md create mode 100644 demo_scripts/simdem/prerequisites/remote.md create mode 100644 demo_scripts/simdem/running/README.md create mode 100644 demo_scripts/simdem/special_commands/README.md create mode 100644 demo_scripts/simdem/syntax/README.md create mode 100644 demo_scripts/simdem/test/README.md create mode 100644 demo_scripts/simdem/tutorial/README.md create mode 100644 demo_scripts/simdem/variables/README.md create mode 100644 demo_scripts/simdem/variables/env.json create mode 100644 demo_scripts/simdem/variables/env.local.json create mode 100644 demo_scripts/test/README.md create mode 100644 demo_scripts/test/directory/README.md create mode 100644 demo_scripts/test/env.json create mode 100644 demo_scripts/test/env.local.json create mode 100644 demo_scripts/test/env.test.json create mode 100644 demo_scripts/test/environment_test.md create mode 100644 demo_scripts/test/prerequisites/README.md create mode 100644 demo_scripts/test/prerequisites/nested_prereq.md create mode 100644 demo_scripts/test/prerequisites/passing_prerequisite.md create mode 100644 demo_scripts/test/remote/README.md create mode 100644 demo_scripts/test/test_plan.txt diff --git a/demo_scripts/README.md b/demo_scripts/README.md new file mode 100644 index 0000000..9e3b09a --- /dev/null +++ b/demo_scripts/README.md @@ -0,0 +1,19 @@ +# Welcome to SimDem + +Is it: + + * Documentation + * An interactive tutorial + * A live demo + * An automated test script + * A Shell script + +## Simdem is Documentation, Tutorials, Demo's and Tests + +It's all of them! + +# Next Steps + + 1. [Hello World Demo](simdem/demo/README.md) + 2. [SimDem Documentation](simdem/README.md) + diff --git a/demo_scripts/env.json b/demo_scripts/env.json new file mode 100644 index 0000000..6fcf682 --- /dev/null +++ b/demo_scripts/env.json @@ -0,0 +1,3 @@ +{ + "PARENT_TEST": "Hello from the parent" +} diff --git a/demo_scripts/simdem/README.md b/demo_scripts/simdem/README.md new file mode 100644 index 0000000..8bb26dc --- /dev/null +++ b/demo_scripts/simdem/README.md @@ -0,0 +1,48 @@ +# Welcome to SimDem + +Is it: + + * Documentation + * An interactive tutorial + * A live demo + * An automated test script + * A Shell script + +## Simdem is Documentation, Tutorials, Demo's and Tests + +It's all of them! + +Simdem allows you to wite a tutorial in markdown format and then run +the commands within it as a simulated demo, interactive tutorial or +even a test script. You can also generate executable shell scripts. + +SimDem reads a script, written in the form of a human readable +Markdown file, and executes the commands within this script on your +behalf. It will even make it look like you are really typing the +commands, which is great if you want to concentrate on explaining what +you are doing but still run the demo live. + +It's easier to describe if you see it working. In fact you are already +in a SimDem. Press a key (other than 'b', we'll look at that shortly) +to "type" a command, once the command has been "typed" hit +a key to execute the command. + +# Next Steps + +Tutorials can branch too, for example you can choose any of the +following paths next: + + 1. [Modes of operation](modes/README.md) + 2. [Hello World Demo](demo/README.md) + 3. [Build a Hello World script](tutorial/README.md) + 4. [Write SimDem documents](syntax/README.md) + 5. [Special Commands](special_commands/README.md) + 6. [Configure your scripts through variables](variables/README.md) + 7. [Write multi-part documents](multipart/README.md) + 8. [Use your documents as interactive tutorials or demos](running/README.md) + 9. [Use your documents as automated tests](test/README.md) + 10. [Build an SimDem container](building/README.md) + + + + diff --git a/demo_scripts/simdem/building/README.md b/demo_scripts/simdem/building/README.md new file mode 100644 index 0000000..2837b1f --- /dev/null +++ b/demo_scripts/simdem/building/README.md @@ -0,0 +1,13 @@ +# Building your own Demo Container + +Create a Dockerfile and add (at least) the following: + + ``` + FROM rgardler/simdem + + COPY my_script_dir demo_scripts + ``` + +# Next Steps + + 1. [SimDem Index](../README.md) diff --git a/demo_scripts/simdem/demo/README.md b/demo_scripts/simdem/demo/README.md new file mode 100644 index 0000000..c46c51a --- /dev/null +++ b/demo_scripts/simdem/demo/README.md @@ -0,0 +1,74 @@ +# Hello World SimDem Demo + +This script is intended to be used to demonstrate the key features of +Simdem. + +When you hit 'spacebar' a command will be displayed and +executed. Actually you can hit almost any key but we recommend +'spacebar' here because we've not told you about the special keys yet +and spacebar is not one of them. + +``` +echo "Hello World" +``` + +That's cool, lets try again: + +``` +echo "It might look like this was typed into the terminal (even more so if you ran SimDem with the '--style simulate' flag, that simulates a person typing), bit it really comes from a markdown file." +``` + +The date command will show that these commands are being executed in real time. + +``` +date +``` + +Results: + +```expected_similarity=0.3 +Sat Mar 12 08:59:01 UTC 2016 +``` + +You can run almost any shell command this way. + +# Special keys + +Although we said "spacebar" above, in reality you can hit almost any +key. There are a few exceptions though: + +## 'd' for description + +Hitting 'd' will print all the text since the last command, that is it +will print the description of the next command to be executed. + +``` +echo "Hitting 'd' now will display the description for this command." +``` + +## 'b' for break + +Hitting 'b' will "break" from the current script. This allows you to +type in commands that are not part of the script. This is particularly +useful when running in demo mode as it alllows you to respond to +questions by entering an unscripted command. + +``` +echo "Give it a go, why not hit 'b' and type 'ls', or some other command" +``` + +NOTE: at the time of writing it is not possible to use interactive +commands or commands. + +# Next Steps + +It's possible to provide a branching point a the end of a script. The +user can select one of a selection of options or they can enter "quit" +(or just "q") to exit SimDem. + + 1. [Write SimDem documents](../syntax/README.md) + 2. [SimDem Index](../README.md) + 3. [Modes of operation](../modes/README.md) + + + diff --git a/demo_scripts/simdem/env.json b/demo_scripts/simdem/env.json new file mode 100644 index 0000000..7a7485b --- /dev/null +++ b/demo_scripts/simdem/env.json @@ -0,0 +1,3 @@ +{ + "TEST": "hello-world" +} diff --git a/demo_scripts/simdem/env.local.json b/demo_scripts/simdem/env.local.json new file mode 100644 index 0000000..e6c7c04 --- /dev/null +++ b/demo_scripts/simdem/env.local.json @@ -0,0 +1,3 @@ +{ + "LOCAL_TEST": "A warm local hello" +} diff --git a/demo_scripts/simdem/modes/README.md b/demo_scripts/simdem/modes/README.md new file mode 100644 index 0000000..9afa34a --- /dev/null +++ b/demo_scripts/simdem/modes/README.md @@ -0,0 +1,95 @@ +# Modes of Operation + +SimDem demos are interactive and can be run in a number of different +modes: + + * Tutorial: Displays the descriptive text of the tutorial and pauses + at code blocks to allow user interaction. + * Learn: similar to Tutorial mode, but users are expected to type + the commands themselves. + * Simulate: Does not display the descriptive text, but pauses at each + code block. When the user hits a key the command is "typed", a + second keypress executes the command. + * Test: Runs the commands and then verifies that the output is + sufficiently similar to the expected results (recorded in the + markdown file) to be considered correct. + * Script: Creates an executable bash script from the document + * Auto: allows any of the above modes to be run but without user + interaction + +## Tutorial Mode + +Tutorial mode is ideal if you are using this as a learning or teaching +tool (see also learn mode below, which suits some learning styles +better. In this mode a description of what you are about to do is +shown on the screen, hit a key to see the command, hit another key to +execute the command. Tutorial mode is the default. + +## Learn mode + +Learn mode is similar to tutorial mode above, however, in learn mode +the user is expected to type each command themselves. Some people find +that this aids recall. + +## Demo (or Simulation) mode + +Demo mode is ideal if you are using this to teach or demonstrate how +to achive the goal. In this mode no descriptive text is shown, instead +when you press a key the next command is "typed", pressing another key +will execute the command. The idea is that you describe what is +happening as the application "types" the command for you. To run in +demo mode use the `--style simulate` command line switch with any +other mode. There is also a default demo mode configuration avilable +with the `demo` command" + +``` +simdem demo +``` + +#### Preparation mode + +In this mode only the preparation (prerequisite) steps are +executed. This is useful for setting up the environment for a +demo. Next time the demo is run all prepration steps will be +skipped. This means that steps that take a long time can be +pre-baked. + +To use this mode use the command 'mode' + +``` +simdem prep +``` + +## Test Mode + +Test mode runs the commands and then verifies that the output is +sufficiently similar to the expected results (recorded in the markdown +file) to be considered correct. To run in test mode use the `--test +yes` switch. For convenience you can use the command `test` to execute +tests with the optimal configuration for automated testing.. + +## Script mode + +Script mode does not execute any of the commands, instead is outputs +an executable bash script that can be run without SimDem. Use the +command `script` to generate the executable script. + +# Unnattended (Auto) Mode + +Each of these modes can be run in auto mode too. This means that the +program does not wait for a keypress before proceeding. This can be +useful if you want to runthe complete script unattended. To run in +automated or unnattended mode use the `--auto true` command line +switch. + +Manual mode is ideal if you would like to manually type the commands, +many people find this helps them remember. It can be useful in the +first few runs, but we still recommend using "demo" mode when doing +live demo's - it's much harder to make a mistake this way. + +# Next Steps + + 1. [Hello World Demo](../demo/README.md) + 2. [SimDem Index](../README.md) + 3. [Write SimDem documents](../syntax/README.md) + 4. [Build a SimDem container](../building/README.md) diff --git a/demo_scripts/simdem/multipart/README.md b/demo_scripts/simdem/multipart/README.md new file mode 100644 index 0000000..18ab03b --- /dev/null +++ b/demo_scripts/simdem/multipart/README.md @@ -0,0 +1,106 @@ +# Multi-Part Demo's + +Most tutorials's will consist of at least three parts, preparation, +main body and cleanup. Many will have multiple staged in the main part +of the tutorial. SimDem is able to provide an interactive menu +allowing users to select which part of the tutorial to work through +next. This is achieved by providing a final section with the title `# +Next Steps`. This section should include a list in which each item +provides a link to a markdown document that contain SimDem scripts +that the user may want to work through next. + +For example, this file is one part of a multi-part document. + +``` +cat $SIMDEM_CWD/README.md +``` + +When executed using SimDem this results in the user being prompted to +select a "next step" (or hit 'q' to quit). If the user selects one of +the scripts it will be executed. + +# Prerequisites + +It is assumed that you have a basic understanding of the various +SimDem execution [modes](../modes/README.md). You should also ensure +you understand the SimDem [document syntax](../syntax). + +# Directory structure + +SimDem projects consist of a root directory and one or more +sub-directories. Project directories will contain at least a +`README.md` file that will be used by default when SimDem is run +against the project. Therefore the minimum directory structure for a +simple tutorial is: + +` +My_SimDem_Tutorial +└── README.md +` + +# Multi-part tutorials + +A more complex project will contain a number of sub-directories +containing tutorials. Tutorial sub-directories will contain at least a +`README.md` file, this is the main file for that tutorial. For example: + +` +My_Complex_SimDem_Tutorial +├── README.md +├── Tutorial_1 +│ └── README.md +├── Tutorial_2 +│ └── README.md +└── Tutorial_3 + └── README.md +` + +## Auto Table of Contents + +If the root of the demo scripts directory does not contain a +`README.md` file then SimDem will create a Table of Contents from all +sub-directories that contain a `README.md` file. This ToC will use the +first line (which should be a heading marked with '# ' at the start) +as the text for the link to the script. This ToC will be displayed as +a 'Next Steps' section, thus users will be able to step into any area +of the available demo's. + +# Other files + +Tutorials may also provide an `env.json` and/or an `env.local.json` +and/or an `env.test.json` file to define environment variables to use +when executing in demo or test mode. + +# Demo Scripts example + +The directory structure for the SimDem demo scripts is: + +``` +tree $SIMDEM_CWD/.. +``` + +Results: Expected Similarity: 0.5 + +``` +demo_scripts/ +├── env.json +├── env.local.json +├── README.md +├── simdem +│ ├── env.json +│ ├── env.local.json +│ └── README.md +└── test + ├── env.json + └── README.md +``` + +# Next Steps + + 1. [Configure your scripts through variables](../variables/README.md) + 2. [SimDem Index](../README.md) + 3. [Use your documents as interactive tutorials or demos](../running/README.md) + 4. [Use your documents as automated tests](../test/README.md) + 5. [Build a SimDem container](../building/README.md) + + diff --git a/demo_scripts/simdem/prerequisites/README.md b/demo_scripts/simdem/prerequisites/README.md new file mode 100644 index 0000000..d0bd278 --- /dev/null +++ b/demo_scripts/simdem/prerequisites/README.md @@ -0,0 +1,96 @@ +# Understanding Prerequisites + +Prerequisite scripts are scripts that must be run in order for another +script to work. When SimDem finds a pre-requisite section it will test +whether the steps have been completed (see validation below). If the +validation tests fail then the code blocks in the pre-requisite script +are executed. + +There aren't really any pre-requisites for this tutorial / +demo. Howeve,r this document is inserted as a pre-requisite so that we +can see how they work. + +# Prerequisites Syntax + +The prerequisites section starts with a heading of `# prerequisites`. + +The body of this section will contain 0 or more links to a script that +should be run ahead of the current one. + +The scripts should appear in the order of required exection in the body. + +# Automatically validating Pre-requisites + +Some pre-requisite steps can take a long time to execute. For this +reason it is possible to provide some validation checks to see if the +pre-requisite step has been completed. These are defined in a section +towards the end of the script, before the next steps section (if one +exists). The validation steps will be executed by SimDem *before* +running the pre-requisite steps, if the tests in that section pass +then there is no need to run the pre-requisites. + +It's easier to explain with an example. + +Imagine we have a prerequisite step that takes 5 seconds, we don't +want to wait 5 seconds only to find that we already completed that +pre-requisite (OK, we know 5 seconds is not long, but it's long enough +to serve for this demo). For this example we will merely sleep for 5 +seconds then touch a file. To validate this prequisite has been +satisfied we will test the modified date of the file, if it has been +modified in the last 5 minutes then the pre-requisite has been +satisfied. + +``` +sleep 5 +echo $SIMDEM_TEMP_DIR +mkdir -p $SIMDEM_TEMP_DIR +touch $SIMDEM_TEMP_DIR/this_file_must_be_modfied_every_minute.txt +``` + +Now we have a set of commands that should be executed as part of this +pre-requisite. In order to use them we simply add a reference to this +file in the pre-requisites section of any other script. + +Any code in a section headed with '# Validation' will be used by +SimDem to test whether the pre-requisites have been satisfied. If +validation tests pass the pre-requisite step will be skipped over, +otherwise the other commands in the script will be executed. + +# Validation + +In order to continue with our example we include some vlaidation steps +in this script. If you have not run through the commands above less +than one minute ago this validation stage will fail. If you are +working through this tutorial now you just executed the above +statements and so the tests here will pass, but if you include this +file as pre-requisite again it may well fail and thus automatically +execute this script. + +For this pre-requisite we need to ensure that the test.txt file has +been updated in the last 5 minutes. If not then we need to run the +commands in this document. If you are running through this document in +SimDem itself then it might be worth going back to the page that calls +this as a pre-requisite, as long as you do this in the next five +minutes you won't come back here. You can do this by selecting +"Understanding SimDem Syntax" in the next steps section. + +``` +find $SIMDEM_TEMP_DIR -name "this_file_must_be_modfied_every_minute.txt" -newermt "1 minutes ago" +``` + +Results: + +``` +/home//.simdem/tmp/this_file_must_be_modfied_every_minute.txt +``` + +# Next Steps + + 1. [Understanding SimDem Syntax](../syntax/script.md) + 2. [Configure your scripts through variables](../variables/script.md) + 3. [Build a Hello World script](../tutorial/script.md) + 4. [SimDem Index](../script.md) + 5. [Write multi-part documents](../multipart/script.md) + 6. [Use your documents as interactive tutorials or demos](../running/script.md) + + diff --git a/demo_scripts/simdem/prerequisites/remote.md b/demo_scripts/simdem/prerequisites/remote.md new file mode 100644 index 0000000..2c3adc8 --- /dev/null +++ b/demo_scripts/simdem/prerequisites/remote.md @@ -0,0 +1,4 @@ +# A remote Prerequsite + +This files is really only for test purposes. See the main +prerequisites script for more details. diff --git a/demo_scripts/simdem/running/README.md b/demo_scripts/simdem/running/README.md new file mode 100644 index 0000000..c430b44 --- /dev/null +++ b/demo_scripts/simdem/running/README.md @@ -0,0 +1,56 @@ +# Running SimDem + +SimDem is packaged as a container, you run it with: + +`docker run -it rgardler/simdem` + +This will run the demo script you are working through now. + +## Adding Your Own Demo Script + +You will likely want to add your own demo script. To do this you can +either build your own container or you can mount a volume which +contains your demo scripts. To mount a volume run: + +`docker run -it -v ~/my_demo_dir:/demo_scripts rgardler/simdem` + +When mounting a directory the script found in the first folder within +the mounted folder will be run. If you want to run a specific demo +within that folder add `run SCRIPT_NAME` to the command, where +SCRIPT_NAME is replaced with the name of the folder containing the +script you want to run. For example: + +`docker run -it -v ~/my_demo_dir:/demo_scripts rgardler/simdem run myscript` + +# Going Off-Script + +You can go off-script if you want to. This is where you should hit 'b' +(for break). You will now be able to type a command to be +run. However, note that at the time of writing the parser for this +command is not very smart, so some commands do not work. In addition +you can't run fully interactive commands this way (so no editors for +example). Go ahead and try it, hit 'b' and type a command, e.g. 'ls'. + +Note that when you hit 'b' you will not see any change in the output, +but you can now start typing freely. + +``` +echo "This is a dummy code block to ensure SimDem pauses in interactive mode" +``` + +# Repeating Commands + +Sometimes a command will need to be run a number of times, for example +it might be monitoring the state of an operation. The easiest way to +repeat a command is simply to press 'r'. + +``` +echo "This is a dummy code block to ensure SimDem pauses in interactive mode" +``` + +# Next Steps + + 1. [Use your documents as automated tests](../test/README.md) + 2. [SimDem Index](../README.md) + 3. [Modes of operation](../modes/README.md) + 4. [Build a SimDem container](../building/README.md) diff --git a/demo_scripts/simdem/special_commands/README.md b/demo_scripts/simdem/special_commands/README.md new file mode 100644 index 0000000..f9dbcb6 --- /dev/null +++ b/demo_scripts/simdem/special_commands/README.md @@ -0,0 +1,37 @@ +# Special Commands + +Some commands will be intercepted by SimDem and handled as special +commands. For example, we might interccept a command to open a browser +at a specific page and handle it differently in a headless CLI +environment to how it is handled in a Web UI environment. + +In fact lets look at that use case as an example. + +## Opening a Browser Tab + +On linux the command `xdg-open` is the accepted way of opening a +browser, therefore it is the accepted way of having such a command in +SimDem script. However, this poses a problem, behoviour of this +command will be different in different UI environments. + +For example, on a headliess CLI environment it will attempt to open +"lynx" or similar text based browsers. Since these are interactive +programs they will not work in SimDem. If running in a desktop +environment it will attempt to open the preferred browser. + +SimDem will intercept this command and handle it appropriately. That +is, in a headless CLI environment it will convert the command to a +"curl -I" command, this at least allows us to ensure that there is a +resposne from the URL provided. When running in a Web UI it will open +a new browser tab (at least at the time of writing, we may decide to +integrate this with the Web UI at some point). + +### In Action + +The command block below contains the `xdg-open` command, depending on +whether you are running in the Web UI or the CLI you will see +different behaviour, as described above. + +``` +xdg-open http://bing.com +``` diff --git a/demo_scripts/simdem/syntax/README.md b/demo_scripts/simdem/syntax/README.md new file mode 100644 index 0000000..8098683 --- /dev/null +++ b/demo_scripts/simdem/syntax/README.md @@ -0,0 +1,120 @@ +# Document syntax + +For the most part SimDem uses standard +[Markdown syntx](https://daringfireball.net/projects/markdown/syntax). There +are a few special strings that can be used to influence how SimDem +works, but even these are intended to be human readable, thus +preventing the need to maintain separate documents for the different +use cases (documentation, tutorial, demo, test and script). + +For example, lets take a look at the start of this file this file: + +``` +head -n 25 README.md +``` + +# Prerequisites + +It is common for a tutorial or demo to have a number of +prerequisites. For example, it's a good idea to understand how these +work, so take a look at our [prerequisites](../prerequisites/README.md) +before proceeding. + +# SimDem Syntax + +There are a small number of SimDem specific items that you should be +aware of. They are detailed in the next few sections. + +## Code Blocks + +In SimDem a code block is marked in exactly the same way it is in +Markdown, that is with three backticks (``````). Unless a Code Block +is marked as a Results block (see next section) it is assumed that +this is executable code. SimDem will execute each line individually. + +For example: + +``` +# This is a code block, this comment will be ignored by SimDem +echo "This command will be 'typed' and executed." +``` + +### Command Limitations + +At the time of writing it is not possible to have interactive +commands. If you try to include such a command SimDem will "hang" as +it waits, silently, for input. + +## Result Blocks + +Result blocks serve two main purposes: + + 1. Help readers of the markdown content understand expected behaviour + 2. Enable SimDem documents to be used as tests + +### Including results blocks for readablity and testing + +When using the script as a web page or printout it is likley that you +will want to include the results. However, when you are running in a +simulation or tutorial mode you will want to rely on the real results +from the current run. You can include a "Results:" section after any +code block. The first code block after this text will be ignored when +running in an interactive mode (such as tutorial or simulation). That +is, in the example below, the `date -u` command will be run +interactively but the `Sat Mar 12 10:09:12 UTC 2016` will only be +included in a static form of the script. + +``` +date -u +``` + +Results: + +```expected_similarity=0.4 +Sat Mar 12 10:09:12 UTC 2016 +``` + +### Modifying Test Accuracy + +When running a script as a test the outputs of the command are +compared to the result block associated with the code block. By +default a similarity of 66%, meaning at least 66% of the characters +are the same, is considered a pass. However, in some cases this is too +high or too low. + +The expected similarity between the command output and the contents of +the result block can be set in the `Results:` header by adding +`Expected similarity: 0.2`, where `0.2` is the similarity desired. + +This can be used to ensure things that have low similarity in the results will pass tests, for example, outputing a date will always result in a different date and thus a much lower expected similarity. + +The date command will prove this is running in real time. + +``` +date +``` + +Results: + +```expected_similarity=0.4 +Sat Mar 12 08:59:01 UTC 2016 +``` + +## Defining Next Steps + +When running in interactive mode it is possible to provide optional +paths for the user to take next. These appear in a section with the +heading "# Next Steps". Note, for this to work as an interactive set +of options this must be the last section in the document. if it is not +the last section then it will be treated like any other heading. + +For example, this document offers next steps options. + +# Next Steps + + 1. [Special Commands](special_commands/README.md) + 2. [Configure your scripts through variables](../variables/README.md) + 3. [Build a Hello World script](../tutorial/README.md) + 4. [SimDem Index](../README.md) + 5. [Write multi-part documents](../multipart/README.md) + 6. [Use your documents as interactive tutorials or demos](../running/README.md) diff --git a/demo_scripts/simdem/test/README.md b/demo_scripts/simdem/test/README.md new file mode 100644 index 0000000..0f00162 --- /dev/null +++ b/demo_scripts/simdem/test/README.md @@ -0,0 +1,89 @@ +# Automated Testing + +When running with the `--test` flag or using the `test` command SimDem +will verify that the output of each command is as expected. It does +this by comparing the output of the command with the `Results:` +section in the script. + +Test mode is very useful in a continuous integration environment. For +example, you can configure your scripts to always use the latest +versions of tooling they depend upon and get early warning when a +change in one of those tools breaks your +scripts. The +[Azure Container Service demos GitHub repository](http://github.com/Azure/acs-demos) shows +this technique in use. + +## Testing in practice + +First, lets take a look at the source of this file. + +``` +cat $SIMDEM_CWD/README.md +``` + +## Running in test mode + +Running in demo mode will not check the results against +expectations. However, running with the `test` command will do so. + +``` +echo "This test is expected to fail" +``` + +Results: + +``` +It fails because the results we have in the script are significantly +different to the output of the command. +``` + +### Similarity tests + +By default a 66% or more match indicates a pass. However, in some +cases a much lower similarity is expected, for example, the output of +`date` will vary considerably each time it is run. In these situations +you can provide an expected similarity as part of the three backticks +that start a code block, for example ```expected_similarity=0.2 which +is low enough for the test to be recorded as a pass. Note, it is +important that you do not insert any spaces in this notation. + +``` +date +``` + +Results: + +```Expected_Similarity=0.2 +Tue Jun 6 15:23:53 UTC 2017 +``` + +## Fast Fail + +The default setting is for SimDem to stop the test run on the first +test failure. This can be overridden by setting the command line flag +`--fastfail` to any value other than `True`. + +## Test Plans + +It is often a good idea to split tests into separate files. SimDem +will allow you to do this by providing a `test_plan.txt` file. Each +line in this file is either a comment (lines starting with '#') or a +filename for a SimDem script to be used in testing. Each of these +files will be concatenated together to create a complete test plan. + +For example, the following example `test_plan.txt` will run all the +code and tests in `preparation/README.md` followed by those in +`main/README.md` and finally those in `cleanup/README.md`. + +` +preparation/README.md +main/README.md +cleanup/README.md +` + +# Next Steps + + 1. [SimDem Index](../README.md) + 2. [Build a Hello World script](../tutorial/README.md) + 3. [Write SimDem documents](../syntax/README.md) + 4. [Configure your scripts through variables](../variables/README.md) diff --git a/demo_scripts/simdem/tutorial/README.md b/demo_scripts/simdem/tutorial/README.md new file mode 100644 index 0000000..e458dbc --- /dev/null +++ b/demo_scripts/simdem/tutorial/README.md @@ -0,0 +1,46 @@ +# Writing a SimDem script + +This document desribes how to write a SimDem script. + +## Install SimDem + +These commands can't be run from within SimDem - danger of turning the +whole world into a recursive simulation of itself ;-). Therefore you +should execute them on your client (Linux, Mac or Windows Subsystem +for Linux). + +`git clone git@github.com:rgardler/simdem.git` +`pushd simdem` +`./install.sh` +`simdem` + +This last command will launch you into the SimDem documentation. If +you haven't reviewed it already you should do so, of particular +importance is the Syntax section. + +## Hello World Script + +Let's build a hello world script: + +``` +mkdir -p hello_world +echo "# Hello World Script" > hello_world/README.md +``` + +## Run the script + +You can't run SimDem within SimDem so if you are reading this from +within SimDem you will need to exit and run `simdem -p hello_world` to +see this in action. + +# Next Steps + +Now you have a working hello world script you are on your own (not +really ask questions / report bugs via the +http://github.com/rgardler/simdem issue tracker). If you feel a little +lost then try one of these documents for guidance: + + 1. [Review SimDem Syntax](../syntx/README.md) + 2. [Understand how to parameterize scripts](../variables/README.md) + 3. [Understand how to make a script a test](../test/README.md) + diff --git a/demo_scripts/simdem/variables/README.md b/demo_scripts/simdem/variables/README.md new file mode 100644 index 0000000..62f4bd2 --- /dev/null +++ b/demo_scripts/simdem/variables/README.md @@ -0,0 +1,168 @@ +# Environment Variables + +In order to use environment variables, you can define one or more +files. These variables are available in every command that is +executed. + +Tutorials can carry `env.json` files in the directory the simdem +command was run and/or in tutorial sub-directories. Files in tutorial +sub-directories will overwrite settings pulled from the project +directory. + +For example, this tutorial defines an 'env.json' in the `simdem` +parent folder and in the `variables` subdirectory that contains this +script. Here is the content from the test subdirectory. + +``` +cat $SIMDEM_CWD/env.json +``` + +Results: + +``` +{ + "NAME": "Hello from the variables subdirectory" +} +``` + +It also defines an 'env.json' file in the `SimDem` root +folder. Assuming you executed the `simdem` command from within that +folder the followin command will display it's content. + +``` +cat env.json +``` + +Results: + +``` +{ + "TEST": "A local hello from the current working directory (where the simdem command was executed)" +} +``` + +Finally, a project may define an `env.local.json` file in the +directory from which the `simdem` command is run. This file is the +last to be loaded and overrides all other values. + +Values are loaded in the following order, the last file to define a + vlaue is the one that "wins". + + - PARENT_OF_SCRIPT_DIR/env.json + - SCRIPT_DIR/env.json + - PARENT_OF_SCRIPT_DIR/env.local.json + - SCRIPT_DIR/env.local.json + - CWD/env.json + - CWD/env.local.json + + +## Interactive Variables + +If you include an environment variable that isn't set, SimDem will +prompt you to give it a value and will add it to the running +environment. If you are running in test mode the variable will be +given a value of 'Dummy vlaue for test'. + +``` +echo $NEW_VARIABLE +``` + +Results: + +``` Expected_Similarity=0 +Enter a value for $NEW_VARIABLE: SimDem + +SimDem + +``` + +### Defining variables can be important + +Because SimDem will interactively ask for values for undefined +variables it is sometimes necessary to first declare a variable to +prevent this action. For example: + +``` +i=0 +for i in {0..4}; do echo "Welcome $i times"; done +``` + +Results: + +``` +Welcome 0 times +Welcome 1 times +Welcome 2 times +Welcome 3 times +Welcome 4 times +``` + +## User provided environment + +Since it is helpful to provide configuration files in published +scripts SimDem also provides a way for users to provide user specific +configurations. So that users can setup their demo's to use private +keys etc. These files are provided in the same way as `env.json` files +(i.e. in the project and tutorial sub-directories) but are called +`env.local.json`. These files take precedence over both project and +tutorial provided files. + +For example, this project provides a local files in both the project +and this tutorial sub-directories. Note that in this case we have +checked them into version control as they are part of the example, +normally they would be added to your local '.gitignore' or equivalent. + +``` +cat $SIMDEM_CWD/../env.local.json +``` + +Results: + +``` +{ + "LOCAL_TEST": "Hello from the local project config" +} +``` + +It also defines an 'env.json' file in the tutorial folder: + +``` +cat $SIMDEM_CWD/env.local.json +``` + +Results: + +``` +{ + "LOCAL_TEST": "A warm local hello" +} +``` + +The file that "wins" is the most local one, that is the one in the tutorial: + +``` +echo $LOCAL_TEST +``` + +Results: + +``` +A warm local hello +``` + +## SimDem Environment Variables + +SimDem provides some information about itself in environment +variables. These are all nameed `SIMDEM_*`. At present the available +variables are: + +``` +env | grep "SIMDEM_" +``` + +# Next Steps + + 1. [Build a SimDem container](../building/README.md) + 2. [SimDem Index](../README.md) + 3. [Use your documents as interactive tutorials or demos](../running/README.md) + 4. [Use your documents as automated tests](../test/README.md) diff --git a/demo_scripts/simdem/variables/env.json b/demo_scripts/simdem/variables/env.json new file mode 100644 index 0000000..b76ccd5 --- /dev/null +++ b/demo_scripts/simdem/variables/env.json @@ -0,0 +1,3 @@ +{ + "NAME": "Hello from the variables subdirectory" +} diff --git a/demo_scripts/simdem/variables/env.local.json b/demo_scripts/simdem/variables/env.local.json new file mode 100644 index 0000000..e6c7c04 --- /dev/null +++ b/demo_scripts/simdem/variables/env.local.json @@ -0,0 +1,3 @@ +{ + "LOCAL_TEST": "A warm local hello" +} diff --git a/demo_scripts/test/README.md b/demo_scripts/test/README.md new file mode 100644 index 0000000..bfe00c2 --- /dev/null +++ b/demo_scripts/test/README.md @@ -0,0 +1,157 @@ +# SimDem Test Script + +This is a simple test script. It runs a number of commands in +succession. This script also lists commands known not to work. + +# Setup + +Ensure the test environment is correctly setup. + +## SimDem version check + +``` +echo $SIMDEM_VERSION +``` + +Results: + +```expected_similarity=0.8 +0.8.2-dev +``` + +## Clean test working files + +Ensure that our working files folder exists and that there are no +residual files from previus test runs. + +``` +echo $SIMDEM_TEMP_DIR +mkdir -p $SIMDEM_TEMP_DIR/test +rm -Rf $SIMDEM_TEMP_DIR/test/* +``` + +# Prerequisites + +Test to see if our prerequesites work. In the setup we cleaned out our +test files. The [prerequisite test script](./prerequisites/README.md) +validates whether the file exists and, if it doesn't it will execute +and create it. + +Each [prerequisite](./prerequisites/README.md) will only be run once, +so even though this partucular prereq appears twice it will only +execute once. This is important when building multi-part tutorials/ +demos where a prereq may be included in more than one part. + +## Validate prerequisite ran + +The prerequisite script should have run and created a `prereq_ran` +file. + +``` +ls $SIMDEM_TEMP_DIR/test +``` + +Results: + +``` +prereq_ran +``` + + +# Directory Check + +``` +head -n 1 README.md +``` + +Results: + +``` Expected_Similarity=0.8 +# SimDem Test Script +``` + +# Simple Echo + +``` +echo "Hello world" +``` + +Results: + +``` +Hello world +``` + +# Code comments + +``` +# This is a comment and should be ignored +echo "This output should be displayed, the comment before this line should be ignored" +``` + +Results: + +``` +This output should be displayed, the comment before this line should be ignored +``` + +# Expected different results + +When we know the results will be different and we want to use them in +tests we need to override the similarity expected by adding +`expected_similarity=x.y` in the start line of the results block: + +``` +date +``` + +Results: + +```expected_Similarity=0.2 +Tue Jun 6 15:23:53 UTC 2017 +``` + +# For Loop + +Because SimDem will interactively ask for values for undefined +variables it is sometimes necessary to first declare a variable to +prevent this action. For example: + +``` +i=0 +for i in {0..10}; do echo "Welcome $i times"; done +``` + +Results: + +``` +Welcome 0 times +Welcome 1 times +Welcome 2 times +Welcome 3 times +Welcome 4 times +Welcome 5 times +Welcome 6 times +Welcome 7 times +Welcome 8 times +Welcome 9 times +Welcome 10 times +``` + +# Stripping ANSI escape sequances + +To make it easier to write scripts we don't want to include ANSI +escape sequences (such as colors and text deocration) in the results +section. SimDem automatically strips these when capturing the results. + +``` +printf "Normal \e[4mUnderlined\e[24m Normal" +``` + +Results: + +```expected_similarity=0.9 +Normal Underlined Normal +``` + + diff --git a/demo_scripts/test/directory/README.md b/demo_scripts/test/directory/README.md new file mode 100644 index 0000000..a8443dc --- /dev/null +++ b/demo_scripts/test/directory/README.md @@ -0,0 +1,26 @@ +# Directory Check + +In this test ensures that the currrent working directory is set +correctly when a test file is loaded as part of the test_plan.txt +file, see [Issue #70](https://github.com/Azure/simdem/issues/70). + +Frst lets check the current working directory, this is useful for +debugging if the test fails. + +``` +pwd +``` + +Since we don't know exactly where this will be stored we need to check +that we can open this file in the test. + +``` +head -n 1 README.md +``` + +Results: + +``` Expected_Similarity=0.8 +# Directory Check +``` + diff --git a/demo_scripts/test/env.json b/demo_scripts/test/env.json new file mode 100644 index 0000000..8bfd5a8 --- /dev/null +++ b/demo_scripts/test/env.json @@ -0,0 +1,4 @@ +{ + "TEST": "Hello from the test script", + "DIR_IN_HOME": "~/should/be/expanded" +} diff --git a/demo_scripts/test/env.local.json b/demo_scripts/test/env.local.json new file mode 100644 index 0000000..3d9ee80 --- /dev/null +++ b/demo_scripts/test/env.local.json @@ -0,0 +1,3 @@ +{ + "TEST": "A local hello from the current working directory (where the simdem command was executed)" +} diff --git a/demo_scripts/test/env.test.json b/demo_scripts/test/env.test.json new file mode 100644 index 0000000..948b4f7 --- /dev/null +++ b/demo_scripts/test/env.test.json @@ -0,0 +1,3 @@ +{ + "TEST_VALUE": "Test value for the test script" +} diff --git a/demo_scripts/test/environment_test.md b/demo_scripts/test/environment_test.md new file mode 100644 index 0000000..c7bde24 --- /dev/null +++ b/demo_scripts/test/environment_test.md @@ -0,0 +1,195 @@ +# Environment tests + +We should be able to retrieve environment variables from the directory +in which the command was given. Note that SimDem provides the +environment variable `SIMDEM_EXEC_DIR` which provides access to this +folder in SimDem scripts should it be necessary. + +``` +cat $SIMDEM_EXEC_DIR/./env.json +``` + +Results: + +``` +{ + "TEST": "Hello from the current working directory (where the simdem command was executed)" +} +``` + +We should also be able to retrieve locallay defined environment +variables from the directory in which the command was given: + + +``` +cat env.local.json +``` + +Results: + +``` +{ + "TEST": "A local hello from the current working directory (where the simdem command was executed)" +} +``` + +There should also be environment variables in the the directory in +which the current script resides. + +``` +cat env.json +``` + +Results: + +``` +{ + "TEST": "Hello from the test script", + "DIR_IN_HOME": "~/should/be/expanded" +} +``` + +Local variables can also be found in the the directory in which the +current script resides. + +``` +cat env.local.json +``` + +Results: + +``` +{ + "TEST": "A local hello from the current working directory (where the simdem command was executed)" +} +``` + +For the `TEST` variable we should have the `env.local.json` value from +the directory in which the application was executed. + +``` +echo $TEST +``` + +Results: + +``` +A local hello from the current working directory (where the simdem command was executed) +``` + +There should be variable definitions in the parent of the +current script directory: + +``` +cat ../env.json +``` + +Results: + +``` +{ + "PARENT_TEST": "Hello from the parent" +} +``` + +Since the value of `PARENT_TEST` is only defined in this file we +should have the value from there: + +``` +echo $PARENT_TEST +``` + +Results: + +``` +Hello from the parent +``` + +## Test Environment + +We can also provide values in `env.test.json` in either the script +directory or the parent of the script directory. If available these +will be loaded first and overwritten by subsequent `env.json` and +`env.local.json` files. For this reason if you want to dorce the user +to provide a value for an environment variable it is important that +you define it as an empty string in `env.json` if a value has been +provided in `env.test.json`. + + +``` +echo $TEST_VALUE +``` + +Results: + +``` +Test value for the test script +``` + +# Replacing variables in special commands + +Some commands cannot be run in headless mode, e.g. `xdg-open`. These +commands will be replaced with an appropriate alternative, +e.g. `curl`. Variables included in such commands will be expanded at +execution time as expected. + +``` +url=http://bing.com +xdg-open $url +``` + +Results: + +```expected_similarity=0.2 +HTTP/1.1 405 Method Not Allowed +Content-Length: 0 +Server: Microsoft-IIS/10.0 +X-MSEdge-Ref: Ref A: D6871D12117C436FAE3762BC8BCA0C29 Ref B: CO1EDGE0415 Ref C: 2017-08-17T15:37:59Z +Date: Thu, 17 Aug 2017 15:37:58 GMT +``` + +Note: this test will only pass when running in headless mode as the +`xdg-open` command will be executed in other environments. + +# Setting new variables in script + +If a script sets a variable during execution this will be recorded in +the SimDem environment. This includes setting to an empty string, this +will prevent SimDem interactively requesting a value for the variable +(or setting a dummt value in test mode). + +``` +new_var="" +echo $new_var +``` + +Results: + +```expected_similarity=0.2 +``` + +# Capturing the output of commands + +``` +CAPTURED_OUTPUT=$(echo foo | sed 's/foo/bar/') +``` + +Captured value is: + +``` +echo $CAPTURED_OUTPUT +``` + +Results: + +``` +bar +``` + +# Processing of Environment Variables + +'~' should be expanded to a home directory (no way to test this). + +``` +echo $DIR_IN_HOME +``` diff --git a/demo_scripts/test/prerequisites/README.md b/demo_scripts/test/prerequisites/README.md new file mode 100644 index 0000000..969792d --- /dev/null +++ b/demo_scripts/test/prerequisites/README.md @@ -0,0 +1,33 @@ +# Test Prerequisites + +This script is not included as part of the test plan, but it should be +executed as part of the root `script.md`. Therefore, because of this +prerequisite there should be a file called `prereq_ran` and another +called `nested_prereq_ran` in the temp directory. + +# Prerequisites + +We should be able to run [nested prerequisites](./nested_prereq.md). + +# Create the test file + +``` +touch $SIMDEM_TEMP_DIR/test/prereq_ran +``` + +# Validation + +If the `prereq_ran` file exists then we don't need to run this +script. In our tests the setup phase removes this file so the +validation test should always fail. + +``` +ls $SIMDEM_TEMP_DIR/test +``` + +Results: + +``` +nested_prereq_ran +prereq_ran +``` diff --git a/demo_scripts/test/prerequisites/nested_prereq.md b/demo_scripts/test/prerequisites/nested_prereq.md new file mode 100644 index 0000000..2876fcf --- /dev/null +++ b/demo_scripts/test/prerequisites/nested_prereq.md @@ -0,0 +1,25 @@ +# Nested Prerequisites + +This prerequisite is executed from the main prerequisite test file. + +# Create the test file + +``` +touch $SIMDEM_TEMP_DIR/test/nested_prereq_ran +``` + +# Validation + +If the `prereq_ran` file exists then we don't need to run this +script. In our tests the setup phase removes this file so the +validation test should always fail. + +``` +ls $SIMDEM_TEMP_DIR/test +``` + +Results: + +``` +nested_prereq_ran +``` diff --git a/demo_scripts/test/prerequisites/passing_prerequisite.md b/demo_scripts/test/prerequisites/passing_prerequisite.md new file mode 100644 index 0000000..3dcd455 --- /dev/null +++ b/demo_scripts/test/prerequisites/passing_prerequisite.md @@ -0,0 +1,28 @@ +# Passing Prerequisite + +This is a dummy prerequisite file that contains a validation step that +will always pass and thus the body of this script will never be +executed. To ensure this is the case when we run tests we have placed +a failing test in the body. + +``` +echo "This test will always fail" +``` + +Results: + +``` +So we can ensure it never runs (the validation step will always pass) +``` + +# Validation + +``` +echo "This validation step is designed to always pass" +``` + +Results: + +``` +This validation step is designed to always pass +``` diff --git a/demo_scripts/test/remote/README.md b/demo_scripts/test/remote/README.md new file mode 100644 index 0000000..f3647df --- /dev/null +++ b/demo_scripts/test/remote/README.md @@ -0,0 +1,33 @@ +# A remote script + +This file will be pulled from GitHub as a remote file to ensure that +execution of remote files works. It is pulled into the test suite in +two ways: + + 1. As a remote prerequisite + 2. AS a remote file in a test plan + +# Create the test file + +``` +mkdir -p $SIMDEM_TEMP_DIR/test +touch $SIMDEM_TEMP_DIR/test/remote_prereq_ran +``` + +# Validation + +If the `remote_prereq_ran` file exists then we don't need to run this +script. In our tests the setup phase removes this file so the +validation test should always fail. + +``` +ls $SIMDEM_TEMP_DIR/test +``` + +Results: + +``` +nested_prereq_ran +prereq_ran +remote_prereq_ran who +``` diff --git a/demo_scripts/test/test_plan.txt b/demo_scripts/test/test_plan.txt new file mode 100644 index 0000000..d482ad4 --- /dev/null +++ b/demo_scripts/test/test_plan.txt @@ -0,0 +1,4 @@ +# SimDem Test Plan +README.md +directory/README.md +environment_test.md