From baf5d4e5fd2a7f9fc31b72a6421d44ef16973350 Mon Sep 17 00:00:00 2001 From: "Zhian N. Kamvar" Date: Mon, 24 Apr 2023 09:07:06 -0700 Subject: [PATCH] [automation] transform lesson to sandpaper --- .editorconfig | 26 ++ .github/workflows/README.md | 198 +++++++++ .github/workflows/pr-close-signal.yaml | 23 + .github/workflows/pr-comment.yaml | 185 ++++++++ .github/workflows/pr-post-remove-branch.yaml | 32 ++ .github/workflows/pr-preflight.yaml | 39 ++ .github/workflows/pr-receive.yaml | 131 ++++++ .github/workflows/sandpaper-main.yaml | 61 +++ .github/workflows/sandpaper-version.txt | 1 + .github/workflows/update-cache.yaml | 125 ++++++ .github/workflows/update-workflows.yaml | 66 +++ .github/workflows/workbench-beta-phase.yml | 60 +++ .gitignore | 55 +++ CODE_OF_CONDUCT.md | 13 + CONTRIBUTING.md | 121 +++++ LICENSE.md | 79 ++++ README.md | 16 +- config.yaml | 90 ++++ episodes/01-intro.md | 209 ++++----- episodes/02-makefiles.md | 417 +++++++++--------- episodes/03-variables.md | 201 +++++---- episodes/04-dependencies.md | 277 ++++++------ episodes/05-patterns.md | 93 ++-- episodes/06-variables.md | 116 +++-- episodes/07-functions.md | 210 ++++----- episodes/08-self-doc.md | 80 ++-- episodes/09-conclusion.md | 268 ++++++----- .../data}/books/LICENSE_TEXTS.md | 0 {data => episodes/data}/books/abyss.txt | 0 {data => episodes/data}/books/isles.txt | 0 {data => episodes/data}/books/last.txt | 0 {data => episodes/data}/books/sierra.txt | 0 .../fig}/02-makefile-challenge.png | Bin {fig => episodes/fig}/02-makefile.png | Bin {fig => episodes/fig}/04-dependencies.png | Bin {fig => episodes/fig}/07-functions.png | Bin .../fig}/09-conclusion-challenge-1.png | Bin {files => episodes/files}/make-lesson.zip | Bin index.md | 44 +- .../instructor-notes.md | 83 ++-- {_extras => learners}/discuss.md | 168 +++---- learners/reference.md | 345 +++++++++++++++ setup.md => learners/setup.md | 24 +- profiles/learner-profiles.md | 5 + reference.md | 362 --------------- site/README.md | 2 + 46 files changed, 2848 insertions(+), 1377 deletions(-) create mode 100644 .editorconfig create mode 100755 .github/workflows/README.md create mode 100755 .github/workflows/pr-close-signal.yaml create mode 100755 .github/workflows/pr-comment.yaml create mode 100755 .github/workflows/pr-post-remove-branch.yaml create mode 100755 .github/workflows/pr-preflight.yaml create mode 100755 .github/workflows/pr-receive.yaml create mode 100755 .github/workflows/sandpaper-main.yaml create mode 100644 .github/workflows/sandpaper-version.txt create mode 100755 .github/workflows/update-cache.yaml create mode 100755 .github/workflows/update-workflows.yaml create mode 100644 .github/workflows/workbench-beta-phase.yml create mode 100644 .gitignore create mode 100644 CODE_OF_CONDUCT.md create mode 100644 CONTRIBUTING.md create mode 100644 LICENSE.md create mode 100644 config.yaml rename {data => episodes/data}/books/LICENSE_TEXTS.md (100%) rename {data => episodes/data}/books/abyss.txt (100%) rename {data => episodes/data}/books/isles.txt (100%) rename {data => episodes/data}/books/last.txt (100%) rename {data => episodes/data}/books/sierra.txt (100%) rename {fig => episodes/fig}/02-makefile-challenge.png (100%) rename {fig => episodes/fig}/02-makefile.png (100%) rename {fig => episodes/fig}/04-dependencies.png (100%) rename {fig => episodes/fig}/07-functions.png (100%) rename {fig => episodes/fig}/09-conclusion-challenge-1.png (100%) rename {files => episodes/files}/make-lesson.zip (100%) rename _extras/guide.md => instructors/instructor-notes.md (85%) rename {_extras => learners}/discuss.md (66%) create mode 100644 learners/reference.md rename setup.md => learners/setup.md (89%) create mode 100644 profiles/learner-profiles.md delete mode 100644 reference.md create mode 100644 site/README.md diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 00000000..5bf4860b --- /dev/null +++ b/.editorconfig @@ -0,0 +1,26 @@ +root = true + +[*] +charset = utf-8 +insert_final_newline = true +trim_trailing_whitespace = true + +[*.md] +indent_size = 2 +indent_style = space +max_line_length = 100 # Please keep this in sync with bin/lesson_check.py! +trim_trailing_whitespace = false # keep trailing spaces in markdown - 2+ spaces are translated to a hard break (
) + +[*.r] +max_line_length = 80 + +[*.py] +indent_size = 4 +indent_style = space +max_line_length = 79 + +[*.sh] +end_of_line = lf + +[Makefile] +indent_style = tab diff --git a/.github/workflows/README.md b/.github/workflows/README.md new file mode 100755 index 00000000..101967e4 --- /dev/null +++ b/.github/workflows/README.md @@ -0,0 +1,198 @@ +# Carpentries Workflows + +This directory contains workflows to be used for Lessons using the {sandpaper} +lesson infrastructure. Two of these workflows require R (`sandpaper-main.yaml` +and `pr-recieve.yaml`) and the rest are bots to handle pull request management. + +These workflows will likely change as {sandpaper} evolves, so it is important to +keep them up-to-date. To do this in your lesson you can do the following in your +R console: + +```r +# Install/Update sandpaper +options(repos = c(carpentries = "https://carpentries.r-universe.dev/", + CRAN = "https://cloud.r-project.org")) +install.packages("sandpaper") + +# update the workflows in your lesson +library("sandpaper") +update_github_workflows() +``` + +Inside this folder, you will find a file called `sandpaper-version.txt`, which +will contain a version number for sandpaper. This will be used in the future to +alert you if a workflow update is needed. + +What follows are the descriptions of the workflow files: + +## Deployment + +### 01 Build and Deploy (sandpaper-main.yaml) + +This is the main driver that will only act on the main branch of the repository. +This workflow does the following: + + 1. checks out the lesson + 2. provisions the following resources + - R + - pandoc + - lesson infrastructure (stored in a cache) + - lesson dependencies if needed (stored in a cache) + 3. builds the lesson via `sandpaper:::ci_deploy()` + +#### Caching + +This workflow has two caches; one cache is for the lesson infrastructure and +the other is for the the lesson dependencies if the lesson contains rendered +content. These caches are invalidated by new versions of the infrastructure and +the `renv.lock` file, respectively. If there is a problem with the cache, +manual invaliation is necessary. You will need maintain access to the repository +and you can either go to the actions tab and [click on the caches button to find +and invalidate the failing cache](https://github.blog/changelog/2022-10-20-manage-caches-in-your-actions-workflows-from-web-interface/) +or by setting the `CACHE_VERSION` secret to the current date (which will +invalidate all of the caches). + +## Updates + +### Setup Information + +These workflows run on a schedule and at the maintainer's request. Because they +create pull requests that update workflows/require the downstream actions to run, +they need a special repository/organization secret token called +`SANDPAPER_WORKFLOW` and it must have the `public_repo` and `workflow` scope. + +This can be an individual user token, OR it can be a trusted bot account. If you +have a repository in one of the official Carpentries accounts, then you do not +need to worry about this token being present because the Carpentries Core Team +will take care of supplying this token. + +If you want to use your personal account: you can go to + +to create a token. Once you have created your token, you should copy it to your +clipboard and then go to your repository's settings > secrets > actions and +create or edit the `SANDPAPER_WORKFLOW` secret, pasting in the generated token. + +If you do not specify your token correctly, the runs will not fail and they will +give you instructions to provide the token for your repository. + +### 02 Maintain: Update Workflow Files (update-workflow.yaml) + +The {sandpaper} repository was designed to do as much as possible to separate +the tools from the content. For local builds, this is absolutely true, but +there is a minor issue when it comes to workflow files: they must live inside +the repository. + +This workflow ensures that the workflow files are up-to-date. The way it work is +to download the update-workflows.sh script from GitHub and run it. The script +will do the following: + +1. check the recorded version of sandpaper against the current version on github +2. update the files if there is a difference in versions + +After the files are updated, if there are any changes, they are pushed to a +branch called `update/workflows` and a pull request is created. Maintainers are +encouraged to review the changes and accept the pull request if the outputs +are okay. + +This update is run ~~weekly or~~ on demand. + +### 03 Maintain: Update Pacakge Cache (update-cache.yaml) + +For lessons that have generated content, we use {renv} to ensure that the output +is stable. This is controlled by a single lockfile which documents the packages +needed for the lesson and the version numbers. This workflow is skipped in +lessons that do not have generated content. + +Because the lessons need to remain current with the package ecosystem, it's a +good idea to make sure these packages can be updated periodically. The +update cache workflow will do this by checking for updates, applying them in a +branch called `updates/packages` and creating a pull request with _only the +lockfile changed_. + +From here, the markdown documents will be rebuilt and you can inspect what has +changed based on how the packages have updated. + +## Pull Request and Review Management + +Because our lessons execute code, pull requests are a secruity risk for any +lesson and thus have security measures associted with them. **Do not merge any +pull requests that do not pass checks and do not have bots commented on them.** + +This series of workflows all go together and are described in the following +diagram and the below sections: + +![Graph representation of a pull request](https://carpentries.github.io/sandpaper/articles/img/pr-flow.dot.svg) + +### Pre Flight Pull Request Validation (pr-preflight.yaml) + +This workflow runs every time a pull request is created and its purpose is to +validate that the pull request is okay to run. This means the following things: + +1. The pull request does not contain modified workflow files +2. If the pull request contains modified workflow files, it does not contain + modified content files (such as a situation where @carpentries-bot will + make an automated pull request) +3. The pull request does not contain an invalid commit hash (e.g. from a fork + that was made before a lesson was transitioned from styles to use the + workbench). + +Once the checks are finished, a comment is issued to the pull request, which +will allow maintainers to determine if it is safe to run the +"Receive Pull Request" workflow from new contributors. + +### Recieve Pull Request (pr-recieve.yaml) + +**Note of caution:** This workflow runs arbitrary code by anyone who creates a +pull request. GitHub has safeguarded the token used in this workflow to have no +priviledges in the repository, but we have taken precautions to protect against +spoofing. + +This workflow is triggered with every push to a pull request. If this workflow +is already running and a new push is sent to the pull request, the workflow +running from the previous push will be cancelled and a new workflow run will be +started. + +The first step of this workflow is to check if it is valid (e.g. that no +workflow files have been modified). If there are workflow files that have been +modified, a comment is made that indicates that the workflow is not run. If +both a workflow file and lesson content is modified, an error will occurr. + +The second step (if valid) is to build the generated content from the pull +request. This builds the content and uploads three artifacts: + +1. The pull request number (pr) +2. A summary of changes after the rendering process (diff) +3. The rendered files (build) + +Because this workflow builds generated content, it follows the same general +process as the `sandpaper-main` workflow with the same caching mechanisms. + +The artifacts produced are used by the next workflow. + +### Comment on Pull Request (pr-comment.yaml) + +This workflow is triggered if the `pr-recieve.yaml` workflow is successful. +The steps in this workflow are: + +1. Test if the workflow is valid and comment the validity of the workflow to the + pull request. +2. If it is valid: create an orphan branch with two commits: the current state + of the repository and the proposed changes. +3. If it is valid: update the pull request comment with the summary of changes + +Importantly: if the pull request is invalid, the branch is not created so any +malicious code is not published. + +From here, the maintainer can request changes from the author and eventually +either merge or reject the PR. When this happens, if the PR was valid, the +preview branch needs to be deleted. + +### Send Close PR Signal (pr-close-signal.yaml) + +Triggered any time a pull request is closed. This emits an artifact that is the +pull request number for the next action + +### Remove Pull Request Branch (pr-post-remove-branch.yaml) + +Tiggered by `pr-close-signal.yaml`. This removes the temporary branch associated with +the pull request (if it was created). diff --git a/.github/workflows/pr-close-signal.yaml b/.github/workflows/pr-close-signal.yaml new file mode 100755 index 00000000..9b129d5d --- /dev/null +++ b/.github/workflows/pr-close-signal.yaml @@ -0,0 +1,23 @@ +name: "Bot: Send Close Pull Request Signal" + +on: + pull_request: + types: + [closed] + +jobs: + send-close-signal: + name: "Send closing signal" + runs-on: ubuntu-latest + if: ${{ github.event.action == 'closed' }} + steps: + - name: "Create PRtifact" + run: | + mkdir -p ./pr + printf ${{ github.event.number }} > ./pr/NUM + - name: Upload Diff + uses: actions/upload-artifact@v3 + with: + name: pr + path: ./pr + diff --git a/.github/workflows/pr-comment.yaml b/.github/workflows/pr-comment.yaml new file mode 100755 index 00000000..bb2eb03c --- /dev/null +++ b/.github/workflows/pr-comment.yaml @@ -0,0 +1,185 @@ +name: "Bot: Comment on the Pull Request" + +# read-write repo token +# access to secrets +on: + workflow_run: + workflows: ["Receive Pull Request"] + types: + - completed + +concurrency: + group: pr-${{ github.event.workflow_run.pull_requests[0].number }} + cancel-in-progress: true + + +jobs: + # Pull requests are valid if: + # - they match the sha of the workflow run head commit + # - they are open + # - no .github files were committed + test-pr: + name: "Test if pull request is valid" + runs-on: ubuntu-latest + if: > + github.event.workflow_run.event == 'pull_request' && + github.event.workflow_run.conclusion == 'success' + outputs: + is_valid: ${{ steps.check-pr.outputs.VALID }} + payload: ${{ steps.check-pr.outputs.payload }} + number: ${{ steps.get-pr.outputs.NUM }} + msg: ${{ steps.check-pr.outputs.MSG }} + steps: + - name: 'Download PR artifact' + id: dl + uses: carpentries/actions/download-workflow-artifact@main + with: + run: ${{ github.event.workflow_run.id }} + name: 'pr' + + - name: "Get PR Number" + if: ${{ steps.dl.outputs.success == 'true' }} + id: get-pr + run: | + unzip pr.zip + echo "NUM=$(<./NR)" >> $GITHUB_OUTPUT + + - name: "Fail if PR number was not present" + id: bad-pr + if: ${{ steps.dl.outputs.success != 'true' }} + run: | + echo '::error::A pull request number was not recorded. The pull request that triggered this workflow is likely malicious.' + exit 1 + - name: "Get Invalid Hashes File" + id: hash + run: | + echo "json<> $GITHUB_OUTPUT + - name: "Check PR" + id: check-pr + if: ${{ steps.dl.outputs.success == 'true' }} + uses: carpentries/actions/check-valid-pr@main + with: + pr: ${{ steps.get-pr.outputs.NUM }} + sha: ${{ github.event.workflow_run.head_sha }} + headroom: 3 # if it's within the last three commits, we can keep going, because it's likely rapid-fire + invalid: ${{ fromJSON(steps.hash.outputs.json)[github.repository] }} + fail_on_error: true + + # Create an orphan branch on this repository with two commits + # - the current HEAD of the md-outputs branch + # - the output from running the current HEAD of the pull request through + # the md generator + create-branch: + name: "Create Git Branch" + needs: test-pr + runs-on: ubuntu-latest + if: ${{ needs.test-pr.outputs.is_valid == 'true' }} + env: + NR: ${{ needs.test-pr.outputs.number }} + permissions: + contents: write + steps: + - name: 'Checkout md outputs' + uses: actions/checkout@v3 + with: + ref: md-outputs + path: built + fetch-depth: 1 + + - name: 'Download built markdown' + id: dl + uses: carpentries/actions/download-workflow-artifact@main + with: + run: ${{ github.event.workflow_run.id }} + name: 'built' + + - if: ${{ steps.dl.outputs.success == 'true' }} + run: unzip built.zip + + - name: "Create orphan and push" + if: ${{ steps.dl.outputs.success == 'true' }} + run: | + cd built/ + git config --local user.email "actions@github.com" + git config --local user.name "GitHub Actions" + CURR_HEAD=$(git rev-parse HEAD) + git checkout --orphan md-outputs-PR-${NR} + git add -A + git commit -m "source commit: ${CURR_HEAD}" + ls -A | grep -v '^.git$' | xargs -I _ rm -r '_' + cd .. + unzip -o -d built built.zip + cd built + git add -A + git commit --allow-empty -m "differences for PR #${NR}" + git push -u --force --set-upstream origin md-outputs-PR-${NR} + + # Comment on the Pull Request with a link to the branch and the diff + comment-pr: + name: "Comment on Pull Request" + needs: [test-pr, create-branch] + runs-on: ubuntu-latest + if: ${{ needs.test-pr.outputs.is_valid == 'true' }} + env: + NR: ${{ needs.test-pr.outputs.number }} + permissions: + pull-requests: write + steps: + - name: 'Download comment artifact' + id: dl + uses: carpentries/actions/download-workflow-artifact@main + with: + run: ${{ github.event.workflow_run.id }} + name: 'diff' + + - if: ${{ steps.dl.outputs.success == 'true' }} + run: unzip ${{ github.workspace }}/diff.zip + + - name: "Comment on PR" + id: comment-diff + if: ${{ steps.dl.outputs.success == 'true' }} + uses: carpentries/actions/comment-diff@main + with: + pr: ${{ env.NR }} + path: ${{ github.workspace }}/diff.md + + # Comment if the PR is open and matches the SHA, but the workflow files have + # changed + comment-changed-workflow: + name: "Comment if workflow files have changed" + needs: test-pr + runs-on: ubuntu-latest + if: ${{ always() && needs.test-pr.outputs.is_valid == 'false' }} + env: + NR: ${{ github.event.workflow_run.pull_requests[0].number }} + body: ${{ needs.test-pr.outputs.msg }} + permissions: + pull-requests: write + steps: + - name: 'Check for spoofing' + id: dl + uses: carpentries/actions/download-workflow-artifact@main + with: + run: ${{ github.event.workflow_run.id }} + name: 'built' + + - name: 'Alert if spoofed' + id: spoof + if: ${{ steps.dl.outputs.success == 'true' }} + run: | + echo 'body<> $GITHUB_ENV + echo '' >> $GITHUB_ENV + echo '## :x: DANGER :x:' >> $GITHUB_ENV + echo 'This pull request has modified workflows that created output. Close this now.' >> $GITHUB_ENV + echo '' >> $GITHUB_ENV + echo 'EOF' >> $GITHUB_ENV + + - name: "Comment on PR" + id: comment-diff + uses: carpentries/actions/comment-diff@main + with: + pr: ${{ env.NR }} + body: ${{ env.body }} + diff --git a/.github/workflows/pr-post-remove-branch.yaml b/.github/workflows/pr-post-remove-branch.yaml new file mode 100755 index 00000000..62c2e98d --- /dev/null +++ b/.github/workflows/pr-post-remove-branch.yaml @@ -0,0 +1,32 @@ +name: "Bot: Remove Temporary PR Branch" + +on: + workflow_run: + workflows: ["Bot: Send Close Pull Request Signal"] + types: + - completed + +jobs: + delete: + name: "Delete branch from Pull Request" + runs-on: ubuntu-latest + if: > + github.event.workflow_run.event == 'pull_request' && + github.event.workflow_run.conclusion == 'success' + permissions: + contents: write + steps: + - name: 'Download artifact' + uses: carpentries/actions/download-workflow-artifact@main + with: + run: ${{ github.event.workflow_run.id }} + name: pr + - name: "Get PR Number" + id: get-pr + run: | + unzip pr.zip + echo "NUM=$(<./NUM)" >> $GITHUB_OUTPUT + - name: 'Remove branch' + uses: carpentries/actions/remove-branch@main + with: + pr: ${{ steps.get-pr.outputs.NUM }} diff --git a/.github/workflows/pr-preflight.yaml b/.github/workflows/pr-preflight.yaml new file mode 100755 index 00000000..d0d7420d --- /dev/null +++ b/.github/workflows/pr-preflight.yaml @@ -0,0 +1,39 @@ +name: "Pull Request Preflight Check" + +on: + pull_request_target: + branches: + ["main"] + types: + ["opened", "synchronize", "reopened"] + +jobs: + test-pr: + name: "Test if pull request is valid" + if: ${{ github.event.action != 'closed' }} + runs-on: ubuntu-latest + outputs: + is_valid: ${{ steps.check-pr.outputs.VALID }} + permissions: + pull-requests: write + steps: + - name: "Get Invalid Hashes File" + id: hash + run: | + echo "json<> $GITHUB_OUTPUT + - name: "Check PR" + id: check-pr + uses: carpentries/actions/check-valid-pr@main + with: + pr: ${{ github.event.number }} + invalid: ${{ fromJSON(steps.hash.outputs.json)[github.repository] }} + fail_on_error: true + - name: "Comment result of validation" + id: comment-diff + if: ${{ always() }} + uses: carpentries/actions/comment-diff@main + with: + pr: ${{ github.event.number }} + body: ${{ steps.check-pr.outputs.MSG }} diff --git a/.github/workflows/pr-receive.yaml b/.github/workflows/pr-receive.yaml new file mode 100755 index 00000000..371ef542 --- /dev/null +++ b/.github/workflows/pr-receive.yaml @@ -0,0 +1,131 @@ +name: "Receive Pull Request" + +on: + pull_request: + types: + [opened, synchronize, reopened] + +concurrency: + group: ${{ github.ref }} + cancel-in-progress: true + +jobs: + test-pr: + name: "Record PR number" + if: ${{ github.event.action != 'closed' }} + runs-on: ubuntu-latest + outputs: + is_valid: ${{ steps.check-pr.outputs.VALID }} + steps: + - name: "Record PR number" + id: record + if: ${{ always() }} + run: | + echo ${{ github.event.number }} > ${{ github.workspace }}/NR # 2022-03-02: artifact name fixed to be NR + - name: "Upload PR number" + id: upload + if: ${{ always() }} + uses: actions/upload-artifact@v3 + with: + name: pr + path: ${{ github.workspace }}/NR + - name: "Get Invalid Hashes File" + id: hash + run: | + echo "json<> $GITHUB_OUTPUT + - name: "echo output" + run: | + echo "${{ steps.hash.outputs.json }}" + - name: "Check PR" + id: check-pr + uses: carpentries/actions/check-valid-pr@main + with: + pr: ${{ github.event.number }} + invalid: ${{ fromJSON(steps.hash.outputs.json)[github.repository] }} + + build-md-source: + name: "Build markdown source files if valid" + needs: test-pr + runs-on: ubuntu-latest + if: ${{ needs.test-pr.outputs.is_valid == 'true' }} + env: + GITHUB_PAT: ${{ secrets.GITHUB_TOKEN }} + RENV_PATHS_ROOT: ~/.local/share/renv/ + CHIVE: ${{ github.workspace }}/site/chive + PR: ${{ github.workspace }}/site/pr + MD: ${{ github.workspace }}/site/built + steps: + - name: "Check Out Main Branch" + uses: actions/checkout@v3 + + - name: "Check Out Staging Branch" + uses: actions/checkout@v3 + with: + ref: md-outputs + path: ${{ env.MD }} + + - name: "Set up R" + uses: r-lib/actions/setup-r@v2 + with: + use-public-rspm: true + install-r: false + + - name: "Set up Pandoc" + uses: r-lib/actions/setup-pandoc@v2 + + - name: "Setup Lesson Engine" + uses: carpentries/actions/setup-sandpaper@main + with: + cache-version: ${{ secrets.CACHE_VERSION }} + + - name: "Setup Package Cache" + uses: carpentries/actions/setup-lesson-deps@main + with: + cache-version: ${{ secrets.CACHE_VERSION }} + + - name: "Validate and Build Markdown" + id: build-site + run: | + sandpaper::package_cache_trigger(TRUE) + sandpaper::validate_lesson(path = '${{ github.workspace }}') + sandpaper:::build_markdown(path = '${{ github.workspace }}', quiet = FALSE) + shell: Rscript {0} + + - name: "Generate Artifacts" + id: generate-artifacts + run: | + sandpaper:::ci_bundle_pr_artifacts( + repo = '${{ github.repository }}', + pr_number = '${{ github.event.number }}', + path_md = '${{ env.MD }}', + path_pr = '${{ env.PR }}', + path_archive = '${{ env.CHIVE }}', + branch = 'md-outputs' + ) + shell: Rscript {0} + + - name: "Upload PR" + uses: actions/upload-artifact@v3 + with: + name: pr + path: ${{ env.PR }} + + - name: "Upload Diff" + uses: actions/upload-artifact@v3 + with: + name: diff + path: ${{ env.CHIVE }} + retention-days: 1 + + - name: "Upload Build" + uses: actions/upload-artifact@v3 + with: + name: built + path: ${{ env.MD }} + retention-days: 1 + + - name: "Teardown" + run: sandpaper::reset_site() + shell: Rscript {0} diff --git a/.github/workflows/sandpaper-main.yaml b/.github/workflows/sandpaper-main.yaml new file mode 100755 index 00000000..e17707ac --- /dev/null +++ b/.github/workflows/sandpaper-main.yaml @@ -0,0 +1,61 @@ +name: "01 Build and Deploy Site" + +on: + push: + branches: + - main + - master + schedule: + - cron: '0 0 * * 2' + workflow_dispatch: + inputs: + name: + description: 'Who triggered this build?' + required: true + default: 'Maintainer (via GitHub)' + reset: + description: 'Reset cached markdown files' + required: false + default: false + type: boolean +jobs: + full-build: + name: "Build Full Site" + runs-on: ubuntu-latest + permissions: + checks: write + contents: write + pages: write + env: + GITHUB_PAT: ${{ secrets.GITHUB_TOKEN }} + RENV_PATHS_ROOT: ~/.local/share/renv/ + steps: + + - name: "Checkout Lesson" + uses: actions/checkout@v3 + + - name: "Set up R" + uses: r-lib/actions/setup-r@v2 + with: + use-public-rspm: true + install-r: false + + - name: "Set up Pandoc" + uses: r-lib/actions/setup-pandoc@v2 + + - name: "Setup Lesson Engine" + uses: carpentries/actions/setup-sandpaper@main + with: + cache-version: ${{ secrets.CACHE_VERSION }} + + - name: "Setup Package Cache" + uses: carpentries/actions/setup-lesson-deps@main + with: + cache-version: ${{ secrets.CACHE_VERSION }} + + - name: "Deploy Site" + run: | + reset <- "${{ github.event.inputs.reset }}" == "true" + sandpaper::package_cache_trigger(TRUE) + sandpaper:::ci_deploy(reset = reset) + shell: Rscript {0} diff --git a/.github/workflows/sandpaper-version.txt b/.github/workflows/sandpaper-version.txt new file mode 100644 index 00000000..4aa09069 --- /dev/null +++ b/.github/workflows/sandpaper-version.txt @@ -0,0 +1 @@ +0.11.15 diff --git a/.github/workflows/update-cache.yaml b/.github/workflows/update-cache.yaml new file mode 100755 index 00000000..676d7424 --- /dev/null +++ b/.github/workflows/update-cache.yaml @@ -0,0 +1,125 @@ +name: "03 Maintain: Update Package Cache" + +on: + workflow_dispatch: + inputs: + name: + description: 'Who triggered this build (enter github username to tag yourself)?' + required: true + default: 'monthly run' + schedule: + # Run every tuesday + - cron: '0 0 * * 2' + +jobs: + preflight: + name: "Preflight Check" + runs-on: ubuntu-latest + outputs: + ok: ${{ steps.check.outputs.ok }} + steps: + - id: check + run: | + if [[ ${{ github.event_name }} == 'workflow_dispatch' ]]; then + echo "ok=true" >> $GITHUB_OUTPUT + echo "Running on request" + # using single brackets here to avoid 08 being interpreted as octal + # https://github.com/carpentries/sandpaper/issues/250 + elif [ `date +%d` -le 7 ]; then + # If the Tuesday lands in the first week of the month, run it + echo "ok=true" >> $GITHUB_OUTPUT + echo "Running on schedule" + else + echo "ok=false" >> $GITHUB_OUTPUT + echo "Not Running Today" + fi + + check_renv: + name: "Check if We Need {renv}" + runs-on: ubuntu-latest + needs: preflight + if: ${{ needs.preflight.outputs.ok == 'true'}} + outputs: + needed: ${{ steps.renv.outputs.exists }} + steps: + - name: "Checkout Lesson" + uses: actions/checkout@v3 + - id: renv + run: | + if [[ -d renv ]]; then + echo "exists=true" >> $GITHUB_OUTPUT + fi + + check_token: + name: "Check SANDPAPER_WORKFLOW token" + runs-on: ubuntu-latest + needs: check_renv + if: ${{ needs.check_renv.outputs.needed == 'true' }} + outputs: + workflow: ${{ steps.validate.outputs.wf }} + repo: ${{ steps.validate.outputs.repo }} + steps: + - name: "validate token" + id: validate + uses: carpentries/actions/check-valid-credentials@main + with: + token: ${{ secrets.SANDPAPER_WORKFLOW }} + + update_cache: + name: "Update Package Cache" + needs: check_token + if: ${{ needs.check_token.outputs.repo== 'true' }} + runs-on: ubuntu-latest + env: + GITHUB_PAT: ${{ secrets.GITHUB_TOKEN }} + RENV_PATHS_ROOT: ~/.local/share/renv/ + steps: + + - name: "Checkout Lesson" + uses: actions/checkout@v3 + + - name: "Set up R" + uses: r-lib/actions/setup-r@v2 + with: + use-public-rspm: true + install-r: false + + - name: "Update {renv} deps and determine if a PR is needed" + id: update + uses: carpentries/actions/update-lockfile@main + with: + cache-version: ${{ secrets.CACHE_VERSION }} + + - name: Create Pull Request + id: cpr + if: ${{ steps.update.outputs.n > 0 }} + uses: carpentries/create-pull-request@main + with: + token: ${{ secrets.SANDPAPER_WORKFLOW }} + delete-branch: true + branch: "update/packages" + commit-message: "[actions] update ${{ steps.update.outputs.n }} packages" + title: "Update ${{ steps.update.outputs.n }} packages" + body: | + :robot: This is an automated build + + This will update ${{ steps.update.outputs.n }} packages in your lesson with the following versions: + + ``` + ${{ steps.update.outputs.report }} + ``` + + :stopwatch: In a few minutes, a comment will appear that will show you how the output has changed based on these updates. + + If you want to inspect these changes locally, you can use the following code to check out a new branch: + + ```bash + git fetch origin update/packages + git checkout update/packages + ``` + + - Auto-generated by [create-pull-request][1] on ${{ steps.update.outputs.date }} + + [1]: https://github.com/carpentries/create-pull-request/tree/main + labels: "type: package cache" + draft: false diff --git a/.github/workflows/update-workflows.yaml b/.github/workflows/update-workflows.yaml new file mode 100755 index 00000000..288bcd13 --- /dev/null +++ b/.github/workflows/update-workflows.yaml @@ -0,0 +1,66 @@ +name: "02 Maintain: Update Workflow Files" + +on: + workflow_dispatch: + inputs: + name: + description: 'Who triggered this build (enter github username to tag yourself)?' + required: true + default: 'weekly run' + clean: + description: 'Workflow files/file extensions to clean (no wildcards, enter "" for none)' + required: false + default: '.yaml' + schedule: + # Run every Tuesday + - cron: '0 0 * * 2' + +jobs: + check_token: + name: "Check SANDPAPER_WORKFLOW token" + runs-on: ubuntu-latest + outputs: + workflow: ${{ steps.validate.outputs.wf }} + repo: ${{ steps.validate.outputs.repo }} + steps: + - name: "validate token" + id: validate + uses: carpentries/actions/check-valid-credentials@main + with: + token: ${{ secrets.SANDPAPER_WORKFLOW }} + + update_workflow: + name: "Update Workflow" + runs-on: ubuntu-latest + needs: check_token + if: ${{ needs.check_token.outputs.workflow == 'true' }} + steps: + - name: "Checkout Repository" + uses: actions/checkout@v3 + + - name: Update Workflows + id: update + uses: carpentries/actions/update-workflows@main + with: + clean: ${{ github.event.inputs.clean }} + + - name: Create Pull Request + id: cpr + if: "${{ steps.update.outputs.new }}" + uses: carpentries/create-pull-request@main + with: + token: ${{ secrets.SANDPAPER_WORKFLOW }} + delete-branch: true + branch: "update/workflows" + commit-message: "[actions] update sandpaper workflow to version ${{ steps.update.outputs.new }}" + title: "Update Workflows to Version ${{ steps.update.outputs.new }}" + body: | + :robot: This is an automated build + + Update Workflows from sandpaper version ${{ steps.update.outputs.old }} -> ${{ steps.update.outputs.new }} + + - Auto-generated by [create-pull-request][1] on ${{ steps.update.outputs.date }} + + [1]: https://github.com/carpentries/create-pull-request/tree/main + labels: "type: template and tools" + draft: false diff --git a/.github/workflows/workbench-beta-phase.yml b/.github/workflows/workbench-beta-phase.yml new file mode 100644 index 00000000..2faa25d9 --- /dev/null +++ b/.github/workflows/workbench-beta-phase.yml @@ -0,0 +1,60 @@ +name: "Deploy to AWS" + +on: + workflow_run: + workflows: ["01 Build and Deploy Site"] + types: + - completed + workflow_dispatch: + +jobs: + preflight: + name: "Preflight Check" + runs-on: ubuntu-latest + outputs: + ok: ${{ steps.check.outputs.ok }} + folder: ${{ steps.check.outputs.folder }} + steps: + - id: check + run: | + if [[ -z "${{ secrets.DISTRIBUTION }}" || -z "${{ secrets.AWS_ACCESS_KEY_ID }}" || -z "${{ secrets.AWS_SECRET_ACCESS_KEY }}" ]]; then + echo ":information_source: No site configured" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo 'To deploy the preview on AWS, you need the `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY` and `DISTRIBUTION` secrets set up' >> $GITHUB_STEP_SUMMARY + else + echo "::set-output name=folder::"$(sed -E 's^.+/(.+)^\1^' <<< ${{ github.repository }}) + echo "::set-output name=ok::true" + fi + + full-build: + name: "Deploy to AWS" + needs: [preflight] + if: ${{ needs.preflight.outputs.ok }} + runs-on: ubuntu-latest + steps: + + - name: "Checkout site folder" + uses: actions/checkout@v3 + with: + ref: 'gh-pages' + path: 'source' + + - name: "Deploy to Bucket" + uses: jakejarvis/s3-sync-action@v0.5.1 + with: + args: --acl public-read --follow-symlinks --delete --exclude '.git/*' + env: + AWS_S3_BUCKET: preview.carpentries.org + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + SOURCE_DIR: 'source' + DEST_DIR: ${{ needs.preflight.outputs.folder }} + + - name: "Invalidate CloudFront" + uses: chetan/invalidate-cloudfront-action@master + env: + PATHS: /* + AWS_REGION: 'us-east-1' + DISTRIBUTION: ${{ secrets.DISTRIBUTION }} + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} diff --git a/.gitignore b/.gitignore new file mode 100644 index 00000000..b8ab7062 --- /dev/null +++ b/.gitignore @@ -0,0 +1,55 @@ +# sandpaper files +episodes/*html +site/* +!site/README.md + +# History files +.Rhistory +.Rapp.history +# Session Data files +.RData +# User-specific files +.Ruserdata +# Example code in package build process +*-Ex.R +# Output files from R CMD build +/*.tar.gz +# Output files from R CMD check +/*.Rcheck/ +# RStudio files +.Rproj.user/ +# produced vignettes +vignettes/*.html +vignettes/*.pdf +# OAuth2 token, see https://github.com/hadley/httr/releases/tag/v0.3 +.httr-oauth +# knitr and R markdown default cache directories +*_cache/ +/cache/ +# Temporary files created by R markdown +*.utf8.md +*.knit.md +# R Environment Variables +.Renviron +# pkgdown site +docs/ +# translation temp files +po/*~ +# renv detritus +renv/sandbox/ +*.pyc +*~ +.DS_Store +.ipynb_checkpoints +.sass-cache +.jekyll-cache/ +.jekyll-metadata +__pycache__ +_site +.Rproj.user +.bundle/ +.vendor/ +vendor/ +.docker-vendor/ +Gemfile.lock +.*history diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md new file mode 100644 index 00000000..f19b8049 --- /dev/null +++ b/CODE_OF_CONDUCT.md @@ -0,0 +1,13 @@ +--- +title: "Contributor Code of Conduct" +--- + +As contributors and maintainers of this project, +we pledge to follow the [The Carpentries Code of Conduct][coc]. + +Instances of abusive, harassing, or otherwise unacceptable behavior +may be reported by following our [reporting guidelines][coc-reporting]. + + +[coc-reporting]: https://docs.carpentries.org/topic_folders/policies/incident-reporting.html +[coc]: https://docs.carpentries.org/topic_folders/policies/code-of-conduct.html diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 00000000..ec44704c --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,121 @@ +## Contributing + +[The Carpentries][cp-site] ([Software Carpentry][swc-site], [Data +Carpentry][dc-site], and [Library Carpentry][lc-site]) are open source +projects, and we welcome contributions of all kinds: new lessons, fixes to +existing material, bug reports, and reviews of proposed changes are all +welcome. + +### Contributor Agreement + +By contributing, you agree that we may redistribute your work under [our +license](LICENSE.md). In exchange, we will address your issues and/or assess +your change proposal as promptly as we can, and help you become a member of our +community. Everyone involved in [The Carpentries][cp-site] agrees to abide by +our [code of conduct](CODE_OF_CONDUCT.md). + +### How to Contribute + +The easiest way to get started is to file an issue to tell us about a spelling +mistake, some awkward wording, or a factual error. This is a good way to +introduce yourself and to meet some of our community members. + +1. If you do not have a [GitHub][github] account, you can [send us comments by + email][contact]. However, we will be able to respond more quickly if you use + one of the other methods described below. + +2. If you have a [GitHub][github] account, or are willing to [create + one][github-join], but do not know how to use Git, you can report problems + or suggest improvements by [creating an issue][issues]. This allows us to + assign the item to someone and to respond to it in a threaded discussion. + +3. If you are comfortable with Git, and would like to add or change material, + you can submit a pull request (PR). Instructions for doing this are + [included below](#using-github). + +Note: if you want to build the website locally, please refer to [The Workbench +documentation][template-doc]. + +### Where to Contribute + +1. If you wish to change this lesson, add issues and pull requests here. +2. If you wish to change the template used for workshop websites, please refer + to [The Workbench documentation][template-doc]. + + +### What to Contribute + +There are many ways to contribute, from writing new exercises and improving +existing ones to updating or filling in the documentation and submitting [bug +reports][issues] about things that do not work, are not clear, or are missing. +If you are looking for ideas, please see [the list of issues for this +repository][repo], or the issues for [Data Carpentry][dc-issues], [Library +Carpentry][lc-issues], and [Software Carpentry][swc-issues] projects. + +Comments on issues and reviews of pull requests are just as welcome: we are +smarter together than we are on our own. **Reviews from novices and newcomers +are particularly valuable**: it's easy for people who have been using these +lessons for a while to forget how impenetrable some of this material can be, so +fresh eyes are always welcome. + +### What *Not* to Contribute + +Our lessons already contain more material than we can cover in a typical +workshop, so we are usually *not* looking for more concepts or tools to add to +them. As a rule, if you want to introduce a new idea, you must (a) estimate how +long it will take to teach and (b) explain what you would take out to make room +for it. The first encourages contributors to be honest about requirements; the +second, to think hard about priorities. + +We are also not looking for exercises or other material that only run on one +platform. Our workshops typically contain a mixture of Windows, macOS, and +Linux users; in order to be usable, our lessons must run equally well on all +three. + +### Using GitHub + +If you choose to contribute via GitHub, you may want to look at [How to +Contribute to an Open Source Project on GitHub][how-contribute]. In brief, we +use [GitHub flow][github-flow] to manage changes: + +1. Create a new branch in your desktop copy of this repository for each + significant change. +2. Commit the change in that branch. +3. Push that branch to your fork of this repository on GitHub. +4. Submit a pull request from that branch to the [upstream repository][repo]. +5. If you receive feedback, make changes on your desktop and push to your + branch on GitHub: the pull request will update automatically. + +NB: The published copy of the lesson is usually in the `main` branch. + +Each lesson has a team of maintainers who review issues and pull requests or +encourage others to do so. The maintainers are community volunteers, and have +final say over what gets merged into the lesson. + +### Other Resources + +The Carpentries is a global organisation with volunteers and learners all over +the world. We share values of inclusivity and a passion for sharing knowledge, +teaching and learning. There are several ways to connect with The Carpentries +community listed at including via social +media, slack, newsletters, and email lists. You can also [reach us by +email][contact]. + +[repo]: https://example.com/FIXME +[contact]: mailto:team@carpentries.org +[cp-site]: https://carpentries.org/ +[dc-issues]: https://github.com/issues?q=user%3Adatacarpentry +[dc-lessons]: https://datacarpentry.org/lessons/ +[dc-site]: https://datacarpentry.org/ +[discuss-list]: https://lists.software-carpentry.org/listinfo/discuss +[github]: https://github.com +[github-flow]: https://guides.github.com/introduction/flow/ +[github-join]: https://github.com/join +[how-contribute]: https://egghead.io/series/how-to-contribute-to-an-open-source-project-on-github +[issues]: https://carpentries.org/help-wanted-issues/ +[lc-issues]: https://github.com/issues?q=user%3ALibraryCarpentry +[swc-issues]: https://github.com/issues?q=user%3Aswcarpentry +[swc-lessons]: https://software-carpentry.org/lessons/ +[swc-site]: https://software-carpentry.org/ +[lc-site]: https://librarycarpentry.org/ +[template-doc]: https://carpentries.github.io/workbench/ diff --git a/LICENSE.md b/LICENSE.md new file mode 100644 index 00000000..7632871f --- /dev/null +++ b/LICENSE.md @@ -0,0 +1,79 @@ +--- +title: "Licenses" +--- + +## Instructional Material + +All Carpentries (Software Carpentry, Data Carpentry, and Library Carpentry) +instructional material is made available under the [Creative Commons +Attribution license][cc-by-human]. The following is a human-readable summary of +(and not a substitute for) the [full legal text of the CC BY 4.0 +license][cc-by-legal]. + +You are free: + +- to **Share**---copy and redistribute the material in any medium or format +- to **Adapt**---remix, transform, and build upon the material + +for any purpose, even commercially. + +The licensor cannot revoke these freedoms as long as you follow the license +terms. + +Under the following terms: + +- **Attribution**---You must give appropriate credit (mentioning that your work + is derived from work that is Copyright (c) The Carpentries and, where + practical, linking to ), provide a [link to the + license][cc-by-human], and indicate if changes were made. You may do so in + any reasonable manner, but not in any way that suggests the licensor endorses + you or your use. + +- **No additional restrictions**---You may not apply legal terms or + technological measures that legally restrict others from doing anything the + license permits. With the understanding that: + +Notices: + +* You do not have to comply with the license for elements of the material in + the public domain or where your use is permitted by an applicable exception + or limitation. +* No warranties are given. The license may not give you all of the permissions + necessary for your intended use. For example, other rights such as publicity, + privacy, or moral rights may limit how you use the material. + +## Software + +Except where otherwise noted, the example programs and other software provided +by The Carpentries are made available under the [OSI][osi]-approved [MIT +license][mit-license]. + +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal in +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies +of the Software, and to permit persons to whom the Software is furnished to do +so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + +## Trademark + +"The Carpentries", "Software Carpentry", "Data Carpentry", and "Library +Carpentry" and their respective logos are registered trademarks of [Community +Initiatives][ci]. + +[cc-by-human]: https://creativecommons.org/licenses/by/4.0/ +[cc-by-legal]: https://creativecommons.org/licenses/by/4.0/legalcode +[mit-license]: https://opensource.org/licenses/mit-license.html +[ci]: https://communityin.org/ +[osi]: https://opensource.org diff --git a/README.md b/README.md index 92149bdf..08e1837b 100644 --- a/README.md +++ b/README.md @@ -1,19 +1,21 @@ [![DOI](https://zenodo.org/badge/DOI/10.5281/zenodo.3265286.svg)](https://doi.org/10.5281/zenodo.3265286) -[![Create a Slack Account with us](https://img.shields.io/badge/Create_Slack_Account-The_Carpentries-071159.svg)](https://swc-slack-invite.herokuapp.com/) - [![Slack Status](https://img.shields.io/badge/Slack_Channel-swc--make-E01563.svg)](https://swcarpentry.slack.com/messages/C9X2YCPT5) +[![Create a Slack Account with us](https://img.shields.io/badge/Create_Slack_Account-The_Carpentries-071159.svg)](https://swc-slack-invite.herokuapp.com/) +[![Slack Status](https://img.shields.io/badge/Slack_Channel-swc--make-E01563.svg)](https://swcarpentry.slack.com/messages/C9X2YCPT5) -make-novice -=========== +# make-novice An introduction to Make using reproducible papers as a motivating example. -Please see for a rendered version +Please see [https://swcarpentry.github.io/make-novice/](https://swcarpentry.github.io/make-novice/) for a rendered version of this material, [the lesson template documentation][lesson-example] for instructions on formatting, building, and submitting material, or run `make` in this directory for a list of helpful commands. Maintainer(s): -* [Gerard Capes][capes-gerard] +- [Gerard Capes][capes-gerard] -[capes-gerard]: https://carpentries.org/instructors/#gcapes [lesson-example]: https://swcarpentry.github.com/lesson-example/ +[capes-gerard]: https://carpentries.org/instructors/#gcapes + + + diff --git a/config.yaml b/config.yaml new file mode 100644 index 00000000..6c410622 --- /dev/null +++ b/config.yaml @@ -0,0 +1,90 @@ +#------------------------------------------------------------ +# Values for this lesson. +#------------------------------------------------------------ + +# Which carpentry is this (swc, dc, lc, or cp)? +# swc: Software Carpentry +# dc: Data Carpentry +# lc: Library Carpentry +# cp: Carpentries (to use for instructor training for instance) +# incubator: The Carpentries Incubator +carpentry: 'swc' + +# Overall title for pages. +title: 'Automation and Make' + +# Date the lesson was created (YYYY-MM-DD, this is empty by default) +created: + +# Comma-separated list of keywords for the lesson +keywords: 'software, data, lesson, The Carpentries' + +# Life cycle stage of the lesson +# possible values: pre-alpha, alpha, beta, stable +life_cycle: 'stable' + +# License of the lesson materials (recommended CC-BY 4.0) +license: 'CC-BY 4.0' + +# Link to the source repository for this lesson +source: 'https://github.com/fishtree-attempt/make-novice/' + +# Default branch of your lesson +branch: 'main' + +# Who to contact if there are any issues +contact: 'team@carpentries.org' + +# Navigation ------------------------------------------------ +# +# Use the following menu items to specify the order of +# individual pages in each dropdown section. Leave blank to +# include all pages in the folder. +# +# Example ------------- +# +# episodes: +# - introduction.md +# - first-steps.md +# +# learners: +# - setup.md +# +# instructors: +# - instructor-notes.md +# +# profiles: +# - one-learner.md +# - another-learner.md + +# Order of episodes in your lesson +episodes: +- 01-intro.md +- 02-makefiles.md +- 03-variables.md +- 04-dependencies.md +- 05-patterns.md +- 06-variables.md +- 07-functions.md +- 08-self-doc.md +- 09-conclusion.md + +# Information for Learners +learners: + +# Information for Instructors +instructors: + +# Learner Profiles +profiles: + +# Customisation --------------------------------------------- +# +# This space below is where custom yaml items (e.g. pinning +# sandpaper and varnish versions) should live + + +url: https://preview.carpentries.org/make-novice +analytics: carpentries +lang: en +workbench-beta: 'true' diff --git a/episodes/01-intro.md b/episodes/01-intro.md index fba089e5..f60006dc 100644 --- a/episodes/01-intro.md +++ b/episodes/01-intro.md @@ -1,25 +1,35 @@ --- -title: "Introduction" +title: Introduction teaching: 25 exercises: 0 -questions: -- "How can I make my results easier to reproduce?" -objectives: -- "Explain what Make is for." -- "Explain why Make differs from shell scripts." -- "Name other popular build tools." -keypoints: -- "Make allows us to specify what depends on what and how to update things that are out of date." --- +::::::::::::::::::::::::::::::::::::::: objectives + +- Explain what Make is for. +- Explain why Make differs from shell scripts. +- Name other popular build tools. + +:::::::::::::::::::::::::::::::::::::::::::::::::: + +:::::::::::::::::::::::::::::::::::::::: questions + +- How can I make my results easier to reproduce? + +:::::::::::::::::::::::::::::::::::::::::::::::::: + Let's imagine that we're interested in testing Zipf's Law in some of our favorite books. -> ## Zipf's Law -> -> The most frequently-occurring word occurs approximately twice as -> often as the second most frequent word. This is [Zipf's Law][zipfs-law]. -{: .callout} +::::::::::::::::::::::::::::::::::::::::: callout + +## Zipf's Law + +The most frequently-occurring word occurs approximately twice as +often as the second most frequent word. This is [Zipf's Law][zipfs-law]. + + +:::::::::::::::::::::::::::::::::::::::::::::::::: We've compiled our raw data i.e. the books we want to analyze and have prepared several Python scripts that together make up our @@ -30,7 +40,7 @@ Let's take quick look at one of the books using the command Our directory has the Python scripts and data files we will be working with: -~~~ +```output |- books | |- abyss.txt | |- isles.txt @@ -40,35 +50,31 @@ Our directory has the Python scripts and data files we will be working with: |- plotcounts.py |- countwords.py |- testzipf.py -~~~ -{: .output} +``` The first step is to count the frequency of each word in a book. For this purpose we will use a python script `countwords.py` which takes two command line arguments. The first argument is the input file (`books/isles.txt`) and the second is the output file that is generated (here `isles.dat`) by processing the input. -~~~ +```bash $ python countwords.py books/isles.txt isles.dat -~~~ -{: .language-bash} +``` Let's take a quick peek at the result. -~~~ +```bash $ head -5 isles.dat -~~~ -{: .language-bash} +``` This shows us the top 5 lines in the output file: -~~~ +```output the 3822 6.7371760973 of 2460 4.33632998414 and 1723 3.03719372466 to 1479 2.60708619778 a 1308 2.30565838181 -~~~ -{: .output} +``` We can see that the file consists of one row per word. Each row shows the word itself, the number of occurrences of that @@ -77,31 +83,28 @@ number of words in the text file. We can do the same thing for a different book: -~~~ +```bash $ python countwords.py books/abyss.txt abyss.dat $ head -5 abyss.dat -~~~ -{: .language-bash} +``` -~~~ +```output the 4044 6.35449402891 and 2807 4.41074795726 of 1907 2.99654305468 a 1594 2.50471401634 to 1515 2.38057825267 -~~~ -{: .output} +``` Let's visualize the results. The script `plotcounts.py` reads in a data file and plots the 10 most frequently occurring words as a text-based bar plot: -~~~ +```bash $ python plotcounts.py isles.dat ascii -~~~ -{: .language-bash} +``` -~~~ +```output the ######################################################################## of ############################################## and ################################ @@ -112,38 +115,33 @@ is ################# that ############ by ########### it ########### -~~~ -{: .output} +``` `plotcounts.py` can also show the plot graphically: -~~~ +```bash $ python plotcounts.py isles.dat show -~~~ -{: .language-bash} +``` Close the window to exit the plot. `plotcounts.py` can also create the plot as an image file (e.g. a PNG file): -~~~ +```bash $ python plotcounts.py isles.dat isles.png -~~~ -{: .language-bash} +``` Finally, let's test Zipf's law for these books: -~~~ +```bash $ python testzipf.py abyss.dat isles.dat -~~~ -{: .language-bash} +``` -~~~ +```output Book First Second Ratio abyss 4044 2807 1.44 isles 3822 2460 1.55 -~~~ -{: .output} +``` So we're not too far off from Zipf's law. @@ -171,7 +169,7 @@ So to reproduce the tasks that we have just done we create a new file named `run_pipeline.sh` in which we place the commands one by one. Using a text editor of your choice (e.g. for nano use the command `nano run_pipeline.sh`) copy and paste the following text and save it. -~~~ +```bash # USAGE: bash run_pipeline.sh # to produce plots for isles and abyss # and the summary table for the Zipf's law tests @@ -184,26 +182,24 @@ python plotcounts.py abyss.dat abyss.png # Generate summary table python testzipf.py abyss.dat isles.dat > results.txt -~~~ -{: .language-bash} +``` Run the script and check that the output is the same as before: -~~~ +```bash $ bash run_pipeline.sh $ cat results.txt -~~~ -{: .language-bash} +``` This shell script solves several problems in computational reproducibility: -1. It explicitly documents our pipeline, - making communication with colleagues (and our future selves) more efficient. -2. It allows us to type a single command, `bash run_pipeline.sh`, to - reproduce the full analysis. -3. It prevents us from _repeating_ typos or mistakes. - You might not get it right the first time, but once you fix something - it'll stay fixed. +1. It explicitly documents our pipeline, + making communication with colleagues (and our future selves) more efficient. +2. It allows us to type a single command, `bash run_pipeline.sh`, to + reproduce the full analysis. +3. It prevents us from *repeating* typos or mistakes. + You might not get it right the first time, but once you fix something + it'll stay fixed. Despite these benefits it has a few shortcomings. @@ -214,7 +210,7 @@ Edit `plotcounts.py` so that the bars are 0.8 units wide instead of 1 unit. `plot_word_counts`.) Now we want to recreate our figures. -We _could_ just `bash run_pipeline.sh` again. +We *could* just `bash run_pipeline.sh` again. That would work, but it could also be a big pain if counting words takes more than a few seconds. The word counting routine hasn't changed; we shouldn't need to recreate @@ -224,12 +220,11 @@ Alternatively, we could manually rerun the plotting for each word-count file. (Experienced shell scripters can make this easier on themselves using a for-loop.) -~~~ +```bash for book in abyss isles; do python plotcounts.py $book.dat $book.png done -~~~ -{: .language-bash} +``` With this approach, however, we don't get many of the benefits of having a shell script in the first place. @@ -237,7 +232,7 @@ we don't get many of the benefits of having a shell script in the first place. Another popular option is to comment out a subset of the lines in `run_pipeline.sh`: -~~~ +```bash # USAGE: bash run_pipeline.sh # to produce plots for isles and abyss # and the summary table for the Zipf's law tests. @@ -252,15 +247,14 @@ python plotcounts.py abyss.dat abyss.png # Generate summary table # This line is also commented out because it doesn't need to be rerun. #python testzipf.py abyss.dat isles.dat > results.txt -~~~ -{: .language-bash} +``` Then, we would run our modified shell script using `bash run_pipeline.sh`. But commenting out these lines, and subsequently uncommenting them, can be a hassle and source of errors in complicated pipelines. -What we really want is an executable _description_ of our pipeline that +What we really want is an executable *description* of our pipeline that allows software to do the tricky part for us: figuring out what steps need to be rerun. @@ -274,15 +268,15 @@ image files, if our text files change. Make can be used for any commands that follow the general pattern of processing files to create new files, for example: -* Run analysis scripts on raw data files to get data files that +- Run analysis scripts on raw data files to get data files that summarize the raw data (e.g. creating files with word counts from book text). -* Run visualization scripts on data files to produce plots +- Run visualization scripts on data files to produce plots (e.g. creating images of word counts). -* Parse and combine text files and plots to create papers. -* Compile source code into executable programs or libraries. +- Parse and combine text files and plots to create papers. +- Compile source code into executable programs or libraries. There are now many build tools available, for example [Apache -ANT][apache-ant], [doit][doit], and [nmake][nmake] for Windows. +ANT][apache-ant], [doit], and [nmake] for Windows. Which is best for you depends on your requirements, intended usage, and operating system. However, they all share the same fundamental concepts as Make. @@ -292,36 +286,47 @@ Autoconf][autoconf] and [CMake][cmake]. Those tools do not run the pipelines directly, but rather generate files for use with the build tools. +::::::::::::::::::::::::::::::::::::::::: callout + +## Why Use Make if it is Almost 40 Years Old? -> ## Why Use Make if it is Almost 40 Years Old? -> -> Make development was started by Stuart Feldman in 1977 as a Bell -> Labs summer intern. Since then it has been undergoing an active -> development and several implementations are available. Since it -> solves a common issue of workflow management, it remains in -> widespread use even today. -> -> Researchers working with legacy codes in C or FORTRAN, which are -> very common in high-performance computing, will, very likely -> encounter Make. -> -> Researchers can use Make for implementing reproducible -> research workflows, automating data analysis and visualisation -> (using Python or R) and combining tables and plots with text to -> produce reports and papers for publication. -> -> Make's fundamental concepts are common across build tools. -{: .callout} +Make development was started by Stuart Feldman in 1977 as a Bell +Labs summer intern. Since then it has been undergoing an active +development and several implementations are available. Since it +solves a common issue of workflow management, it remains in +widespread use even today. + +Researchers working with legacy codes in C or FORTRAN, which are +very common in high-performance computing, will, very likely +encounter Make. + +Researchers can use Make for implementing reproducible +research workflows, automating data analysis and visualisation +(using Python or R) and combining tables and plots with text to +produce reports and papers for publication. + +Make's fundamental concepts are common across build tools. + + +:::::::::::::::::::::::::::::::::::::::::::::::::: [GNU Make][gnu-make] is a free-libre, fast, [well-documented](gnu-make-documentation), and very popular Make implementation. From now on, we will focus on it, and when we say Make, we mean GNU Make. -[autoconf]: http://www.gnu.org/software/autoconf/autoconf.html -[apache-ant]: http://ant.apache.org/ -[cmake]: http://www.cmake.org/ -[doit]: http://pydoit.org/ -[gnu-make]: http://www.gnu.org/software/make/ +[zipfs-law]: https://en.wikipedia.org/wiki/Zipf%27s_law +[apache-ant]: https://ant.apache.org/ +[doit]: https://pydoit.org/ [nmake]: https://docs.microsoft.com/en-us/cpp/build/reference/nmake-reference -[zipfs-law]: http://en.wikipedia.org/wiki/Zipf%27s_law -[gnu-make-documentation]: https://www.gnu.org/software/make/manual/html_node/index.html +[autoconf]: https://www.gnu.org/software/autoconf/autoconf.html +[cmake]: https://www.cmake.org/ +[gnu-make]: https://www.gnu.org/software/make/ + + +:::::::::::::::::::::::::::::::::::::::: keypoints + +- Make allows us to specify what depends on what and how to update things that are out of date. + +:::::::::::::::::::::::::::::::::::::::::::::::::: + + diff --git a/episodes/02-makefiles.md b/episodes/02-makefiles.md index 8402cccf..b582019f 100644 --- a/episodes/02-makefiles.md +++ b/episodes/02-makefiles.md @@ -1,57 +1,59 @@ --- -title: "Makefiles" +title: Makefiles teaching: 30 exercises: 10 -questions: -- "How do I write a simple Makefile?" -objectives: -- "Recognize the key parts of a Makefile, rules, targets, dependencies and actions." -- "Write a simple Makefile." -- "Run Make from the shell." -- "Explain when and why to mark targets as `.PHONY`." -- "Explain constraints on dependencies." -keypoints: -- "Use `#` for comments in Makefiles." -- "Write rules as `target: dependencies`." -- "Specify update actions in a tab-indented block under the rule." -- "Use `.PHONY` to mark targets that don't correspond to files." --- +::::::::::::::::::::::::::::::::::::::: objectives + +- Recognize the key parts of a Makefile, rules, targets, dependencies and actions. +- Write a simple Makefile. +- Run Make from the shell. +- Explain when and why to mark targets as `.PHONY`. +- Explain constraints on dependencies. + +:::::::::::::::::::::::::::::::::::::::::::::::::: + +:::::::::::::::::::::::::::::::::::::::: questions + +- How do I write a simple Makefile? + +:::::::::::::::::::::::::::::::::::::::::::::::::: + Create a file, called `Makefile`, with the following content: -~~~ +```make # Count words. isles.dat : books/isles.txt python countwords.py books/isles.txt isles.dat -~~~ -{: .language-make} +``` -This is a [build file]({{ page.root }}/reference.html#build-file), which for -Make is called a [Makefile]({{ page.root }}/reference.html#makefile) - a file +This is a [build file](../learners/reference.md#build-file), which for +Make is called a [Makefile](../learners/reference.md#makefile) - a file executed by Make. Note how it resembles one of the lines from our shell script. Let us go through each line in turn: -* `#` denotes a *comment*. Any text from `#` to the end of the line is +- `#` denotes a *comment*. Any text from `#` to the end of the line is ignored by Make but could be very helpful for anyone reading your Makefile. -* `isles.dat` is a [target]({{ page.root }}/reference.html#target), a file to be +- `isles.dat` is a [target](../learners/reference.md#target), a file to be created, or built. -* `books/isles.txt` is a [dependency]({{ page.root }}/reference.html#dependency), a +- `books/isles.txt` is a [dependency](../learners/reference.md#dependency), a file that is needed to build or update the target. Targets can have zero or more dependencies. -* A colon, `:`, separates targets from dependencies. -* `python countwords.py books/isles.txt isles.dat` is an - [action]({{ page.root }}/reference.html#action), a command to run to build or +- A colon, `:`, separates targets from dependencies. +- `python countwords.py books/isles.txt isles.dat` is an + [action](../learners/reference.md#action), a command to run to build or update the target using the dependencies. Targets can have zero or more actions. These actions form a recipe to build the target from its dependencies and are executed similarly to a shell script. -* Actions are indented using a single TAB character, *not* 8 spaces. This +- Actions are indented using a single TAB character, *not* 8 spaces. This is a legacy of Make's 1970's origins. If the difference between - spaces and a TAB character isn’t obvious in your editor, try moving + spaces and a TAB character isn't obvious in your editor, try moving your cursor from one side of the TAB to the other. It should jump four or more spaces. -* Together, the target, dependencies, and actions form a - [rule]({{ page.root }}/reference.html#rule). +- Together, the target, dependencies, and actions form a + [rule](../learners/reference.md#rule). Our rule above describes how to build the target `isles.dat` using the action `python countwords.py` and the dependency `books/isles.txt`. @@ -63,67 +65,63 @@ requires `books/isles.txt` - is now made explicit by Make's syntax. Let's first ensure we start from scratch and delete the `.dat` and `.png` files we created earlier: -~~~ +```bash $ rm *.dat *.png -~~~ -{: .language-bash} +``` By default, Make looks for a Makefile, called `Makefile`, and we can run Make as follows: -~~~ +```bash $ make -~~~ -{: .language-bash} +``` By default, Make prints out the actions it executes: -~~~ +```output python countwords.py books/isles.txt isles.dat -~~~ -{: .output} +``` If we see, -~~~ +```error Makefile:3: *** missing separator. Stop. -~~~ -{: .error} +``` then we have used a space instead of a TAB characters to indent one of our actions. Let's see if we got what we expected. -~~~ +```bash head -5 isles.dat -~~~ -{: .language-bash} +``` The first 5 lines of `isles.dat` should look exactly like before. -> ## Makefiles Do Not Have to be Called `Makefile` -> -> We don't have to call our Makefile `Makefile`. However, if we call it -> something else we need to tell Make where to find it. This we can do -> using `-f` flag. For example, if our Makefile is named `MyOtherMakefile`: -> -> ~~~ -> $ make -f MyOtherMakefile -> ~~~ -> {: .language-bash} -> -> -> Sometimes, the suffix `.mk` will be used to identify Makefiles that -> are not called `Makefile` e.g. `install.mk`, `common.mk` etc. -{: .callout} +::::::::::::::::::::::::::::::::::::::::: callout + +## Makefiles Do Not Have to be Called `Makefile` + +We don't have to call our Makefile `Makefile`. However, if we call it +something else we need to tell Make where to find it. This we can do +using `-f` flag. For example, if our Makefile is named `MyOtherMakefile`: + +```bash +$ make -f MyOtherMakefile +``` + +Sometimes, the suffix `.mk` will be used to identify Makefiles that +are not called `Makefile` e.g. `install.mk`, `common.mk` etc. + + +:::::::::::::::::::::::::::::::::::::::::::::::::: When we re-run our Makefile, Make now informs us that: -~~~ +```output make: `isles.dat' is up to date. -~~~ -{: .output} +``` This is because our target, `isles.dat`, has now been created, and Make will not create it again. To see how this works, let's pretend to @@ -131,40 +129,35 @@ update one of the text files. Rather than opening the file in an editor, we can use the shell `touch` command to update its timestamp (which would happen if we did edit the file): -~~~ +```bash $ touch books/isles.txt -~~~ -{: .language-bash} +``` If we compare the timestamps of `books/isles.txt` and `isles.dat`, -~~~ +```bash $ ls -l books/isles.txt isles.dat -~~~ -{: .language-bash} +``` then we see that `isles.dat`, the target, is now older than `books/isles.txt`, its dependency: -~~~ +```output -rw-r--r-- 1 mjj Administ 323972 Jun 12 10:35 books/isles.txt -rw-r--r-- 1 mjj Administ 182273 Jun 12 09:58 isles.dat -~~~ -{: .output} +``` If we run Make again, -~~~ +```bash $ make -~~~ -{: .language-bash} +``` then it recreates `isles.dat`: -~~~ +```output python countwords.py books/isles.txt isles.dat -~~~ -{: .output} +``` When it is asked to build a target, Make checks the 'last modification time' of both the target and its dependencies. If any dependency has @@ -172,115 +165,112 @@ been updated since the target, then the actions are re-run to update the target. Using this approach, Make knows to only rebuild the files that, either directly or indirectly, depend on the file that changed. This is called an [incremental -build]({{ page.root }}/reference.html#incremental-build). +build](../learners/reference.md#incremental-build). + +::::::::::::::::::::::::::::::::::::::::: callout -> ## Makefiles as Documentation -> -> By explicitly recording the inputs to and outputs from steps in our -> analysis and the dependencies between files, Makefiles act as a type -> of documentation, reducing the number of things we have to remember. -{: .callout} +## Makefiles as Documentation + +By explicitly recording the inputs to and outputs from steps in our +analysis and the dependencies between files, Makefiles act as a type +of documentation, reducing the number of things we have to remember. + + +:::::::::::::::::::::::::::::::::::::::::::::::::: Let's add another rule to the end of `Makefile`: -~~~ +```make abyss.dat : books/abyss.txt python countwords.py books/abyss.txt abyss.dat -~~~ -{: .language-make} +``` If we run Make, -~~~ +```bash $ make -~~~ -{: .language-bash} +``` then we get: -~~~ +```output make: `isles.dat' is up to date. -~~~ -{: .output} +``` Nothing happens because Make attempts to build the first target it finds in the Makefile, the -[default target]({{ page.root }}/reference.html#default-target), which is +[default target](../learners/reference.md#default-target), which is `isles.dat` which is already up-to-date. We need to explicitly tell Make we want to build `abyss.dat`: -~~~ +```bash $ make abyss.dat -~~~ -{: .language-bash} +``` Now, we get: -~~~ +```output python countwords.py books/abyss.txt abyss.dat -~~~ -{: .output} - -> ## "Up to Date" Versus "Nothing to be Done" -> -> If we ask Make to build a file that already exists and is up to -> date, then Make informs us that: -> -> ~~~ -> make: `isles.dat' is up to date. -> ~~~ -> {: .output} -> -> If we ask Make to build a file that exists but for which there is -> no rule in our Makefile, then we get message like: -> -> ~~~ -> $ make countwords.py -> ~~~ -> {: .language-bash} -> -> ~~~ -> make: Nothing to be done for `countwords.py'. -> ~~~ -> {: .output} -> -> `up to date` means that the Makefile has a rule with one or more actions -> whose target is the name of a file (or directory) and the file is up to date. -> -> `Nothing to be done` means that -> the file exists but either : -> - the Makefile has no rule for it, or -> - the Makefile has a rule for it, but that rule has no actions -{: .callout} +``` + +::::::::::::::::::::::::::::::::::::::::: callout + +## "Up to Date" Versus "Nothing to be Done" + +If we ask Make to build a file that already exists and is up to +date, then Make informs us that: + +```output +make: `isles.dat' is up to date. +``` + +If we ask Make to build a file that exists but for which there is +no rule in our Makefile, then we get message like: + +```bash +$ make countwords.py +``` + +```output +make: Nothing to be done for `countwords.py'. +``` + +`up to date` means that the Makefile has a rule with one or more actions +whose target is the name of a file (or directory) and the file is up to date. +`Nothing to be done` means that +the file exists but either : + +- the Makefile has no rule for it, or +- the Makefile has a rule for it, but that rule has no actions + + +:::::::::::::::::::::::::::::::::::::::::::::::::: We may want to remove all our data files so we can explicitly recreate them all. We can introduce a new target, and associated rule, to do this. We will call it `clean`, as this is a common name for rules that delete auto-generated files, like our `.dat` files: -~~~ +```make clean : rm -f *.dat -~~~ -{: .language-make} +``` This is an example of a rule that has no dependencies. `clean` has no dependencies on any `.dat` file as it makes no sense to create these just to remove them. We just want to remove the data files whether or not they exist. If we run Make and specify this target, -~~~ +```bash $ make clean -~~~ -{: .language-bash} +``` then we get: -~~~ +```output rm -f *.dat -~~~ -{: .output} +``` There is no actual thing built called `clean`. Rather, it is a short-hand that we can use to execute a useful sequence of @@ -288,59 +278,53 @@ actions. Such targets, though very useful, can lead to problems. For example, let us recreate our data files, create a directory called `clean`, then run Make: -~~~ +```bash $ make isles.dat abyss.dat $ mkdir clean $ make clean -~~~ -{: .language-bash} +``` We get: -~~~ +```output make: `clean' is up to date. -~~~ -{: .output} +``` Make finds a file (or directory) called `clean` and, as its `clean` rule has no dependencies, assumes that `clean` has been built and is up-to-date and so does not execute the rule's actions. As we are using `clean` as a short-hand, we need to tell Make to always execute this rule if we run `make clean`, by telling Make that this is a -[phony target]({{ page.root }}/reference.html#phony-target), that it does not build +[phony target](../learners/reference.md#phony-target), that it does not build anything. This we do by marking the target as `.PHONY`: -~~~ +```make .PHONY : clean clean : rm -f *.dat -~~~ -{: .language-make} +``` If we run Make, -~~~ +```bash $ make clean -~~~ -{: .language-bash} +``` then we get: -~~~ +```output rm -f *.dat -~~~ -{: .output} +``` We can add a similar command to create all the data files. We can put this at the top of our Makefile so that it is the [default -target]({{ page.root }}/reference.html#default-target), which is executed by default +target](../learners/reference.md#default-target), which is executed by default if no target is given to the `make` command: -~~~ +```make .PHONY : dats dats : isles.dat abyss.dat -~~~ -{: .language-make} +``` This is an example of a rule that has dependencies that are targets of other rules. When Make runs, it will check to see if the dependencies @@ -348,52 +332,52 @@ exist and, if not, will see if rules are available that will create these. If such rules exist it will invoke these first, otherwise Make will raise an error. -> ## Dependencies -> -> The order of rebuilding dependencies is arbitrary. You should not -> assume that they will be built in the order in which they are -> listed. -> -> Dependencies must form a directed acyclic graph. A target cannot -> depend on a dependency which itself, or one of its dependencies, -> depends on that target. -{: .callout} +::::::::::::::::::::::::::::::::::::::::: callout + +## Dependencies + +The order of rebuilding dependencies is arbitrary. You should not +assume that they will be built in the order in which they are +listed. + +Dependencies must form a directed acyclic graph. A target cannot +depend on a dependency which itself, or one of its dependencies, +depends on that target. + + +:::::::::::::::::::::::::::::::::::::::::::::::::: This rule (`dats`) is also an example of a rule that has no actions. It is used purely to trigger the build of its dependencies, if needed. If we run, -~~~ +```bash $ make dats -~~~ -{: .language-bash} +``` then Make creates the data files: -~~~ +```output python countwords.py books/isles.txt isles.dat python countwords.py books/abyss.txt abyss.dat -~~~ -{: .output} +``` If we run `make dats` again, then Make will see that the dependencies (`isles.dat` and `abyss.dat`) are already up to date. Given the target `dats` has no actions, there is `nothing to be done`: -~~~ + +```bash $ make dats -~~~ -{: .language-bash} +``` -~~~ +```output make: Nothing to be done for `dats'. -~~~ -{: .output} - +``` Our Makefile now looks like this: -~~~ +```make # Count words. .PHONY : dats dats : isles.dat abyss.dat @@ -407,33 +391,52 @@ abyss.dat : books/abyss.txt .PHONY : clean clean : rm -f *.dat -~~~ -{: .language-make} +``` The following figure shows a graph of the dependencies embodied within our Makefile, involved in building the `dats` target: -![Dependencies represented within the Makefile](../fig/02-makefile.png "Dependencies represented within the Makefile") - -> ## Write Two New Rules -> -> 1. Write a new rule for `last.dat`, created from `books/last.txt`. -> 2. Update the `dats` rule with this target. -> 3. Write a new rule for `results.txt`, which creates the summary -> table. The rule needs to: -> * Depend upon each of the three `.dat` files. -> * Invoke the action `python testzipf.py abyss.dat isles.dat last.dat > results.txt`. -> 4. Put this rule at the top of the Makefile so that it is the default target. -> 5. Update `clean` so that it removes `results.txt`. -> -> The starting Makefile is [here]({{ page.root }}/code/02-makefile/Makefile). -> -> > ## Solution -> > See [this file]({{ page.root }}/code/02-makefile-challenge/Makefile) for a solution. -> {: .solution} -{: .challenge} +![](fig/02-makefile.png "Dependencies represented within the Makefile"){alt='Dependencies represented within the Makefile'} + +::::::::::::::::::::::::::::::::::::::: challenge + +## Write Two New Rules + +1. Write a new rule for `last.dat`, created from `books/last.txt`. +2. Update the `dats` rule with this target. +3. Write a new rule for `results.txt`, which creates the summary + table. The rule needs to: + - Depend upon each of the three `.dat` files. + - Invoke the action `python testzipf.py abyss.dat isles.dat last.dat > results.txt`. +4. Put this rule at the top of the Makefile so that it is the default target. +5. Update `clean` so that it removes `results.txt`. + +The starting Makefile is [here](code/02-makefile/Makefile). + +::::::::::::::: solution + +## Solution + +See [this file](code/02-makefile-challenge/Makefile) for a solution. + + + +::::::::::::::::::::::::: + +:::::::::::::::::::::::::::::::::::::::::::::::::: The following figure shows the dependencies embodied within our Makefile, involved in building the `results.txt` target: -![results.txt dependencies represented within the Makefile](../fig/02-makefile-challenge.png "results.txt dependencies represented within the Makefile") +![](fig/02-makefile-challenge.png "results.txt dependencies represented within the Makefile"){alt='results.txt dependencies represented within the Makefile'} + +:::::::::::::::::::::::::::::::::::::::: keypoints + +- Use `#` for comments in Makefiles. +- Write rules as `target: dependencies`. +- Specify update actions in a tab-indented block under the rule. +- Use `.PHONY` to mark targets that don't correspond to files. + +:::::::::::::::::::::::::::::::::::::::::::::::::: + + diff --git a/episodes/03-variables.md b/episodes/03-variables.md index 6fbfc16a..ace6bf48 100644 --- a/episodes/03-variables.md +++ b/episodes/03-variables.md @@ -1,22 +1,26 @@ --- -title: "Automatic Variables" +title: Automatic Variables teaching: 10 exercises: 5 -questions: -- "How can I abbreviate the rules in my Makefiles?" -objectives: -- "Use Make automatic variables to remove duplication in a Makefile." -- "Explain why shell wildcards in dependencies can cause problems." -keypoints: -- "Use `$@` to refer to the target of the current rule." -- "Use `$^` to refer to the dependencies of the current rule." -- "Use `$<` to refer to the first dependency of the current rule." --- +::::::::::::::::::::::::::::::::::::::: objectives + +- Use Make automatic variables to remove duplication in a Makefile. +- Explain why shell wildcards in dependencies can cause problems. + +:::::::::::::::::::::::::::::::::::::::::::::::::: + +:::::::::::::::::::::::::::::::::::::::: questions + +- How can I abbreviate the rules in my Makefiles? + +:::::::::::::::::::::::::::::::::::::::::::::::::: + After the exercise at the end of the previous episode, our Makefile looked like this: -~~~ +```make # Generate summary table. results.txt : isles.dat abyss.dat last.dat python testzipf.py abyss.dat isles.dat last.dat > results.txt @@ -38,8 +42,7 @@ last.dat : books/last.txt clean : rm -f *.dat rm -f results.txt -~~~ -{: .language-make} +``` Our Makefile has a lot of duplication. For example, the names of text files and data files are repeated in many places throughout the @@ -47,50 +50,51 @@ Makefile. Makefiles are a form of code and, in any code, repeated code can lead to problems e.g. we rename a data file in one part of the Makefile but forget to rename it elsewhere. -> ## D.R.Y. (Don't Repeat Yourself) -> -> In many programming languages, the bulk of the language features are -> there to allow the programmer to describe long-winded computational -> routines as short, expressive, beautiful code. Features in Python -> or R or Java, such as user-defined variables and functions are useful in -> part because they mean we don't have to write out (or think about) -> all of the details over and over again. This good habit of writing -> things out only once is known as the "Don't Repeat Yourself" -> principle or D.R.Y. -{: .callout} +::::::::::::::::::::::::::::::::::::::::: callout + +## D.R.Y. (Don't Repeat Yourself) + +In many programming languages, the bulk of the language features are +there to allow the programmer to describe long-winded computational +routines as short, expressive, beautiful code. Features in Python +or R or Java, such as user-defined variables and functions are useful in +part because they mean we don't have to write out (or think about) +all of the details over and over again. This good habit of writing +things out only once is known as the "Don't Repeat Yourself" +principle or D.R.Y. + + +:::::::::::::::::::::::::::::::::::::::::::::::::: Let us set about removing some of the repetition from our Makefile. In our `results.txt` rule we duplicate the data file names and the name of the results file name: -~~~ +```make results.txt : isles.dat abyss.dat last.dat python testzipf.py abyss.dat isles.dat last.dat > results.txt -~~~ -{: .language-make} +``` Looking at the results file name first, we can replace it in the action with `$@`: -~~~ +```make results.txt : isles.dat abyss.dat last.dat python testzipf.py abyss.dat isles.dat last.dat > $@ -~~~ -{: .language-make} +``` `$@` is a Make -[automatic variable]({{ page.root }}/reference.html#automatic-variable) +[automatic variable](../learners/reference.md#automatic-variable) which means 'the target of the current rule'. When Make is run it will replace this variable with the target name. We can replace the dependencies in the action with `$^`: -~~~ +```make results.txt : isles.dat abyss.dat last.dat python testzipf.py $^ > $@ -~~~ -{: .language-make} +``` `$^` is another automatic variable which means 'all the dependencies of the current rule'. Again, when Make is run it will replace this @@ -98,55 +102,59 @@ variable with the dependencies. Let's update our text files and re-run our rule: -~~~ +```bash $ touch books/*.txt $ make results.txt -~~~ -{: .language-bash} +``` We get: -~~~ +```output python countwords.py books/isles.txt isles.dat python countwords.py books/abyss.txt abyss.dat python countwords.py books/last.txt last.dat python testzipf.py isles.dat abyss.dat last.dat > results.txt -~~~ -{: .output} - - -> ## Update Dependencies -> -> What will happen if you now execute: -> -> ~~~ -> $ touch *.dat -> $ make results.txt -> ~~~ -> {: .language-bash} -> -> 1. nothing -> 2. all files recreated -> 3. only `.dat` files recreated -> 4. only `results.txt` recreated -> -> > ## Solution -> > `4.` Only `results.txt` recreated. -> > -> > The rules for `*.dat` are not executed because their corresponding `.txt` files -> > haven't been modified. -> > -> > If you run: -> > -> > ~~~ -> > $ touch books/*.txt -> > $ make results.txt -> > ~~~ -> > {: .language-bash} -> > -> > you will find that the `.dat` files as well as `results.txt` are recreated. -> {: .solution} -{: .challenge} +``` + +::::::::::::::::::::::::::::::::::::::: challenge + +## Update Dependencies + +What will happen if you now execute: + +```bash +$ touch *.dat +$ make results.txt +``` + +1. nothing +2. all files recreated +3. only `.dat` files recreated +4. only `results.txt` recreated + +::::::::::::::: solution + +## Solution + +`4.` Only `results.txt` recreated. + +The rules for `*.dat` are not executed because their corresponding `.txt` files +haven't been modified. + +If you run: + +```bash +$ touch books/*.txt +$ make results.txt +``` + +you will find that the `.dat` files as well as `results.txt` are recreated. + + + +::::::::::::::::::::::::: + +:::::::::::::::::::::::::::::::::::::::::::::::::: As we saw, `$^` means 'all the dependencies of the current rule'. This works well for `results.txt` as its action treats all the dependencies @@ -162,16 +170,35 @@ one input file to be named when it is invoked. Make provides an automatic variable for this, `$<` which means 'the first dependency of the current rule'. -> ## Rewrite `.dat` Rules to Use Automatic Variables -> -> Rewrite each `.dat` rule to use the automatic variables `$@` ('the -> target of the current rule') and `$<` ('the first dependency of the -> current rule'). -> [This file]({{ page.root }}/code/03-variables/Makefile) contains -> the Makefile immediately before the challenge. -> -> > ## Solution -> > See [this file]({{ page.root }}/code/03-variables-challenge/Makefile) -> > for a solution to this challenge. -> {: .solution} -{: .challenge} +::::::::::::::::::::::::::::::::::::::: challenge + +## Rewrite `.dat` Rules to Use Automatic Variables + +Rewrite each `.dat` rule to use the automatic variables `$@` ('the +target of the current rule') and `$<` ('the first dependency of the +current rule'). +[This file](code/03-variables/Makefile) contains +the Makefile immediately before the challenge. + +::::::::::::::: solution + +## Solution + +See [this file](code/03-variables-challenge/Makefile) +for a solution to this challenge. + + + +::::::::::::::::::::::::: + +:::::::::::::::::::::::::::::::::::::::::::::::::: + +:::::::::::::::::::::::::::::::::::::::: keypoints + +- Use `$@` to refer to the target of the current rule. +- Use `$^` to refer to the dependencies of the current rule. +- Use `$<` to refer to the first dependency of the current rule. + +:::::::::::::::::::::::::::::::::::::::::::::::::: + + diff --git a/episodes/04-dependencies.md b/episodes/04-dependencies.md index 6de48c5c..2658db22 100644 --- a/episodes/04-dependencies.md +++ b/episodes/04-dependencies.md @@ -1,20 +1,25 @@ --- -title: "Dependencies on Data and Code" +title: Dependencies on Data and Code teaching: 15 exercises: 5 -questions: -- "How can I write a Makefile to update things when my scripts have changed rather than my input files?" -objectives: -- "Output files are a product not only of input files but of the scripts or code that created the output files." -- "Recognize and avoid false dependencies." -keypoints: -- "Make results depend on processing scripts as well as data files." -- "Dependencies are transitive: if A depends on B and B depends on C, a change to C will indirectly trigger an update to A." --- +::::::::::::::::::::::::::::::::::::::: objectives + +- Output files are a product not only of input files but of the scripts or code that created the output files. +- Recognize and avoid false dependencies. + +:::::::::::::::::::::::::::::::::::::::::::::::::: + +:::::::::::::::::::::::::::::::::::::::: questions + +- How can I write a Makefile to update things when my scripts have changed rather than my input files? + +:::::::::::::::::::::::::::::::::::::::::::::::::: + Our Makefile now looks like this: -~~~ +```make # Generate summary table. results.txt : isles.dat abyss.dat last.dat python testzipf.py $^ > $@ @@ -36,8 +41,7 @@ last.dat : books/last.txt clean : rm -f *.dat rm -f results.txt -~~~ -{: .language-make} +``` Our data files are produced using not only the input text files but also the script `countwords.py` that processes the text files and creates the @@ -46,12 +50,11 @@ summary data or removing an existing one) results in changes to the `.dat` files it outputs. So, let's pretend to edit `countwords.py`, using `touch`, and re-run Make: -~~~ +```bash $ make dats $ touch countwords.py $ make dats -~~~ -{: .language-bash} +``` Nothing happens! Though we've updated `countwords.py` our data files are not updated because our rules for creating `.dat` files don't @@ -60,7 +63,7 @@ record any dependencies on `countwords.py`. We need to add `countwords.py` as a dependency of each of our data files also: -~~~ +```make isles.dat : books/isles.txt countwords.py python countwords.py $< $@ @@ -69,39 +72,37 @@ abyss.dat : books/abyss.txt countwords.py last.dat : books/last.txt countwords.py python countwords.py $< $@ -~~~ -{: .language-make} +``` If we pretend to edit `countwords.py` and re-run Make, -~~~ +```bash $ touch countwords.py $ make dats -~~~ -{: .language-bash} +``` then we get: -~~~ +```output python countwords.py books/isles.txt isles.dat python countwords.py books/abyss.txt abyss.dat python countwords.py books/last.txt last.dat -~~~ -{: .output} - -> ## Dry run -> -> `make` can show the commands it will execute without actually running them if we pass the `-n` flag: -> -> ~~~ -> $ touch countwords.py -> $ make -n dats -> ~~~ -> {: .language-bash} -> -> This gives the same output to the screen as without the `-n` flag, but the commands are not actually run. Using this 'dry-run' mode is a good way to check that you have set up your Makefile properly before actually running the commands in it. -> -{: .callout} +``` + +::::::::::::::::::::::::::::::::::::::::: callout + +## Dry run + +`make` can show the commands it will execute without actually running them if we pass the `-n` flag: + +```bash +$ touch countwords.py +$ make -n dats +``` + +This gives the same output to the screen as without the `-n` flag, but the commands are not actually run. Using this 'dry-run' mode is a good way to check that you have set up your Makefile properly before actually running the commands in it. + +:::::::::::::::::::::::::::::::::::::::::::::::::: The following figure shows a graph of the dependencies, that are involved in building the target `results.txt`. Notice the recently @@ -109,35 +110,37 @@ added dependencies `countwords.py` and `testzipf.py`. This is how the Makefile should look after completing the rest of the exercises in this episode. -![results.txt dependencies after adding countwords.py and testzipf.py as dependencies](../fig/04-dependencies.png "results.txt dependencies after adding countwords.py and testzipf.py as dependencies") +![](fig/04-dependencies.png "results.txt dependencies after adding countwords.py and testzipf.py as dependencies"){alt='results.txt dependencies after adding countwords.py and testzipf.py as dependencies'} + +::::::::::::::::::::::::::::::::::::::::: callout -> ## Why Don't the `.txt` Files Depend on `countwords.py`? -> -> `.txt` files are input files and as such have no dependencies. To make these -> depend on `countwords.py` would introduce a [false -> dependency]({{ page.root }}/reference.html#false-dependency) which is not desirable. -{: .callout} +## Why Don't the `.txt` Files Depend on `countwords.py`? + +`.txt` files are input files and as such have no dependencies. To make these +depend on `countwords.py` would introduce a [false +dependency](../learners/reference.md#false-dependency) which is not desirable. + + +:::::::::::::::::::::::::::::::::::::::::::::::::: Intuitively, we should also add `countwords.py` as a dependency for `results.txt`, because the final table should be rebuilt if we remake the `.dat` files. However, it turns out we don't have to do that! Let's see what happens to `results.txt` when we update `countwords.py`: -~~~ +```bash $ touch countwords.py $ make results.txt -~~~ -{: .language-bash} +``` then we get: -~~~ +```output python countwords.py books/abyss.txt abyss.dat python countwords.py books/isles.txt isles.dat python countwords.py books/last.txt last.dat python testzipf.py abyss.dat isles.dat last.dat > results.txt -~~~ -{: .output} +``` The whole pipeline is triggered, even the creation of the `results.txt` file! To understand this, note that according to the @@ -149,72 +152,82 @@ newer than the target file (`results.txt`) and thus it recreates subset of the files in the pipeline triggers rerunning the appropriate downstream steps. -> ## Updating One Input File -> -> What will happen if you now execute: -> -> ~~~ -> $ touch books/last.txt -> $ make results.txt -> ~~~ -> {: .language-bash} -> -> 1. only `last.dat` is recreated -> 2. all `.dat` files are recreated -> 3. only `last.dat` and `results.txt` are recreated -> 4. all `.dat` and `results.txt` are recreated -> -> > ## Solution -> > `3.` only `last.dat` and `results.txt` are recreated. -> > -> > Follow the dependency tree to understand the answer(s). -> {: .solution} -{: .challenge} - -> ## `testzipf.py` as a Dependency of `results.txt`. -> -> What would happen if you added `testzipf.py` as dependency of `results.txt`, and why? -> -> > ## Solution -> > -> > If you change the rule for the `results.txt` file like this: -> > -> > ~~~ -> > results.txt : isles.dat abyss.dat last.dat testzipf.py -> > python testzipf.py $^ > $@ -> > ~~~ -> > {: .language-make} -> > -> > `testzipf.py` becomes a part of `$^`, thus the command becomes -> > -> > ~~~ -> > python testzipf.py abyss.dat isles.dat last.dat testzipf.py > results.txt -> > ~~~ -> > {: .language-bash} -> > -> > This results in an error from `testzipf.py` as it tries to parse the -> > script as if it were a `.dat` file. Try this by running: -> > -> > ~~~ -> > $ make results.txt -> > ~~~ -> > {: .language-bash} -> > -> > You'll get -> > -> > ~~~ -> > python testzipf.py abyss.dat isles.dat last.dat testzipf.py > results.txt -> > Traceback (most recent call last): -> > File "testzipf.py", line 19, in -> > counts = load_word_counts(input_file) -> > File "path/to/testzipf.py", line 39, in load_word_counts -> > counts.append((fields[0], int(fields[1]), float(fields[2]))) -> > IndexError: list index out of range -> > make: *** [results.txt] Error 1 -> > ~~~ -> > {: .error} -> {: .solution} -{: .challenge} +::::::::::::::::::::::::::::::::::::::: challenge + +## Updating One Input File + +What will happen if you now execute: + +```bash +$ touch books/last.txt +$ make results.txt +``` + +1. only `last.dat` is recreated +2. all `.dat` files are recreated +3. only `last.dat` and `results.txt` are recreated +4. all `.dat` and `results.txt` are recreated + +::::::::::::::: solution + +## Solution + +`3.` only `last.dat` and `results.txt` are recreated. + +Follow the dependency tree to understand the answer(s). + + + +::::::::::::::::::::::::: + +:::::::::::::::::::::::::::::::::::::::::::::::::: + +::::::::::::::::::::::::::::::::::::::: challenge + +## `testzipf.py` as a Dependency of `results.txt`. + +What would happen if you added `testzipf.py` as dependency of `results.txt`, and why? + +::::::::::::::: solution + +## Solution + +If you change the rule for the `results.txt` file like this: + +```make +results.txt : isles.dat abyss.dat last.dat testzipf.py + python testzipf.py $^ > $@ +``` + +`testzipf.py` becomes a part of `$^`, thus the command becomes + +```bash +python testzipf.py abyss.dat isles.dat last.dat testzipf.py > results.txt +``` + +This results in an error from `testzipf.py` as it tries to parse the +script as if it were a `.dat` file. Try this by running: + +```bash +$ make results.txt +``` + +You'll get + +```error +python testzipf.py abyss.dat isles.dat last.dat testzipf.py > results.txt +Traceback (most recent call last): + File "testzipf.py", line 19, in + counts = load_word_counts(input_file) + File "path/to/testzipf.py", line 39, in load_word_counts + counts.append((fields[0], int(fields[1]), float(fields[2]))) +IndexError: list index out of range +make: *** [results.txt] Error 1 +``` + +::::::::::::::::::::::::: + +:::::::::::::::::::::::::::::::::::::::::::::::::: We still have to add the `testzipf.py` script as dependency to `results.txt`. @@ -224,14 +237,26 @@ we need to make a couple of small changes so that we can still use automatic var We'll move `testzipf.py` to be the first dependency and then edit the action so that we pass all the dependencies as arguments to python using `$^`. -~~~ +```make results.txt : testzipf.py isles.dat abyss.dat last.dat python $^ > $@ -~~~ -{: .language-make} - -> ## Where We Are -> -> [This Makefile]({{ page.root }}/code/04-dependencies/Makefile) -> contains everything done so far in this topic. -{: .callout} +``` + +::::::::::::::::::::::::::::::::::::::::: callout + +## Where We Are + +[This Makefile](code/04-dependencies/Makefile) +contains everything done so far in this topic. + + +:::::::::::::::::::::::::::::::::::::::::::::::::: + +:::::::::::::::::::::::::::::::::::::::: keypoints + +- Make results depend on processing scripts as well as data files. +- Dependencies are transitive: if A depends on B and B depends on C, a change to C will indirectly trigger an update to A. + +:::::::::::::::::::::::::::::::::::::::::::::::::: + + diff --git a/episodes/05-patterns.md b/episodes/05-patterns.md index c7e19dee..470677f6 100644 --- a/episodes/05-patterns.md +++ b/episodes/05-patterns.md @@ -1,29 +1,33 @@ --- -title: "Pattern Rules" +title: Pattern Rules teaching: 10 exercises: 0 -questions: -- "How can I define rules to operate on similar files?" -objectives: -- "Write Make pattern rules." -keypoints: -- "Use the wildcard `%` as a placeholder in targets and dependencies." -- "Use the special variable `$*` to refer to matching sets of files in actions." --- +::::::::::::::::::::::::::::::::::::::: objectives + +- Write Make pattern rules. + +:::::::::::::::::::::::::::::::::::::::::::::::::: + +:::::::::::::::::::::::::::::::::::::::: questions + +- How can I define rules to operate on similar files? + +:::::::::::::::::::::::::::::::::::::::::::::::::: + Our Makefile still has repeated content. The rules for each `.dat` file are identical apart from the text and data file names. We can replace these rules with a single [pattern -rule]({{ page.root }}/reference.html#pattern-rule) which can be used to build any +rule](../learners/reference.md#pattern-rule) which can be used to build any `.dat` file from a `.txt` file in `books/`: -~~~ +```make %.dat : countwords.py books/%.txt python $^ $@ -~~~ -{: .language-make} +``` -`%` is a Make [wildcard]({{ page.root }}/reference.html#wildcard), +`%` is a Make [wildcard](../learners/reference.md#wildcard), matching any number of any characters. This rule can be interpreted as: @@ -33,47 +37,47 @@ and run `python [the dependencies] [the target]`." If we re-run Make, -~~~ +```bash $ make clean $ make dats -~~~ -{: .language-bash} +``` then we get: -~~~ +```output python countwords.py books/isles.txt isles.dat python countwords.py books/abyss.txt abyss.dat python countwords.py books/last.txt last.dat -~~~ -{: .output} +``` Note that we can still use Make to build individual `.dat` targets as before, and that our new rule will work no matter what stem is being matched. -``` +```bash $ make sierra.dat ``` -{: .language-bash} which gives the output below: -``` +```output python countwords.py books/sierra.txt sierra.dat ``` -{: .output} -> ## Using Make Wildcards -> -> The Make `%` wildcard can only be used in a target and in its -> dependencies. It cannot be used in actions. In actions, you may -> however use `$*`, which will be replaced by the stem with which -> the rule matched. -{: .callout} +::::::::::::::::::::::::::::::::::::::::: callout + +## Using Make Wildcards + +The Make `%` wildcard can only be used in a target and in its +dependencies. It cannot be used in actions. In actions, you may +however use `$*`, which will be replaced by the stem with which +the rule matched. + + +:::::::::::::::::::::::::::::::::::::::::::::::::: Our Makefile is now much shorter and cleaner: -~~~ +```make # Generate summary table. results.txt : testzipf.py isles.dat abyss.dat last.dat python $^ > $@ @@ -89,12 +93,23 @@ dats : isles.dat abyss.dat last.dat clean : rm -f *.dat rm -f results.txt -~~~ -{: .language-make} - -> ## Where We Are -> -> [This Makefile]({{ page.root }}/code/05-patterns/Makefile) -> contains all of our work so far. -{: .callout} +``` + +::::::::::::::::::::::::::::::::::::::::: callout + +## Where We Are + +[This Makefile](code/05-patterns/Makefile) +contains all of our work so far. + + +:::::::::::::::::::::::::::::::::::::::::::::::::: + +:::::::::::::::::::::::::::::::::::::::: keypoints + +- Use the wildcard `%` as a placeholder in targets and dependencies. +- Use the special variable `$*` to refer to matching sets of files in actions. + +:::::::::::::::::::::::::::::::::::::::::::::::::: + diff --git a/episodes/06-variables.md b/episodes/06-variables.md index 7b697863..dd854542 100644 --- a/episodes/06-variables.md +++ b/episodes/06-variables.md @@ -1,43 +1,46 @@ --- -title: "Variables" +title: Variables teaching: 15 exercises: 5 -questions: -- "How can I eliminate redundancy in my Makefiles?" -objectives: -- "Use variables in a Makefile." -- "Explain the benefits of decoupling configuration from computation." -keypoints: -- "Define variables by assigning values to names." -- "Reference variables using `$(...)`." --- +::::::::::::::::::::::::::::::::::::::: objectives + +- Use variables in a Makefile. +- Explain the benefits of decoupling configuration from computation. + +:::::::::::::::::::::::::::::::::::::::::::::::::: + +:::::::::::::::::::::::::::::::::::::::: questions + +- How can I eliminate redundancy in my Makefiles? + +:::::::::::::::::::::::::::::::::::::::::::::::::: + Despite our efforts, our Makefile still has repeated content, i.e. the name of our script -- `countwords.py`, and the program we use to run it -- `python`. If we renamed our script we'd have to update our Makefile in multiple places. -We can introduce a Make [variable]({{ page.root }}/reference.html#variable) (called a -[macro]({{ page.root }}/reference.html#macro) in some versions of Make) to hold our +We can introduce a Make [variable](../learners/reference.md#variable) (called a +[macro](../learners/reference.md#macro) in some versions of Make) to hold our script name: -~~~ +```make COUNT_SRC=countwords.py -~~~ -{: .language-make} +``` -This is a variable [assignment]({{ page.root }}/reference.html#assignment) - +This is a variable [assignment](../learners/reference.md#assignment) - `COUNT_SRC` is assigned the value `countwords.py`. We can do the same thing with the interpreter language used to run the script: -~~~ +```make LANGUAGE=python -~~~ -{: .language-make} +``` `$(...)` tells Make to replace a variable with its value when Make -is run. This is a variable [reference]({{ page.root }}/reference.html#reference). At +is run. This is a variable [reference](../learners/reference.md#reference). At any place where we want to use the value of a variable we have to write it, or reference it, in this way. @@ -51,50 +54,56 @@ change how our script is run (e.g. we might want to use a different version of Python and need to change `python` to `python2` -- or we might want to rewrite the script using another language (e.g. switch from Python to R)). -> ## Use Variables -> -> Update `Makefile` so that the `%.dat` rule -> references the variable `COUNT_SRC`. -> Then do the same for the `testzipf.py` script -> and the `results.txt` rule, -> using `ZIPF_SRC` as the variable name. -> -> > ## Solution -> > [This Makefile]({{ page.root }}/code/06-variables-challenge/Makefile) -> > contains a solution to this challenge. -> {: .solution} -{: .challenge} +::::::::::::::::::::::::::::::::::::::: challenge + +## Use Variables + +Update `Makefile` so that the `%.dat` rule +references the variable `COUNT_SRC`. +Then do the same for the `testzipf.py` script +and the `results.txt` rule, +using `ZIPF_SRC` as the variable name. + +::::::::::::::: solution + +## Solution + +[This Makefile](code/06-variables-challenge/Makefile) +contains a solution to this challenge. + + + +::::::::::::::::::::::::: + +:::::::::::::::::::::::::::::::::::::::::::::::::: We place variables at the top of a Makefile so they are easy to find and modify. Alternatively, we can pull them out into a new file that just holds variable definitions (i.e. delete them from the original Makefile). Let us create `config.mk`: -~~~ +```make # Count words script. LANGUAGE=python COUNT_SRC=countwords.py # Test Zipf's rule ZIPF_SRC=testzipf.py -~~~ -{: .language-make} +``` We can then import `config.mk` into `Makefile` using: -~~~ +```make include config.mk -~~~ -{: .language-make} +``` We can re-run Make to see that everything still works: -~~~ +```bash $ make clean $ make dats $ make results.txt -~~~ -{: .language-bash} +``` We have separated the configuration of our Makefile from its rules -- the parts that do all the work. If we want to change our script name @@ -103,9 +112,22 @@ our source code in `Makefile`. Decoupling code from configuration in this way is good programming practice, as it promotes more modular, flexible and reusable code. -> ## Where We Are -> -> [This Makefile]({{ page.root }}/code/06-variables/Makefile) -> and [its accompanying `config.mk`]({{ page.root }}/code/06-variables/config.mk) -> contain all of our work so far. -{: .callout} +::::::::::::::::::::::::::::::::::::::::: callout + +## Where We Are + +[This Makefile](code/06-variables/Makefile) +and [its accompanying `config.mk`](code/06-variables/config.mk) +contain all of our work so far. + + +:::::::::::::::::::::::::::::::::::::::::::::::::: + +:::::::::::::::::::::::::::::::::::::::: keypoints + +- Define variables by assigning values to names. +- Reference variables using `$(...)`. + +:::::::::::::::::::::::::::::::::::::::::::::::::: + + diff --git a/episodes/07-functions.md b/episodes/07-functions.md index c9def7dd..d9c1cc34 100644 --- a/episodes/07-functions.md +++ b/episodes/07-functions.md @@ -1,20 +1,24 @@ --- -title: "Functions" +title: Functions teaching: 20 exercises: 5 -questions: -- "How *else* can I eliminate redundancy in my Makefiles?" -objectives: -- "Write Makefiles that use functions to match and transform sets of files." -keypoints: -- "Make is actually a small programming language with many built-in functions." -- "Use `wildcard` function to get lists of files matching a pattern." -- "Use `patsubst` function to rewrite file names." --- +::::::::::::::::::::::::::::::::::::::: objectives + +- Write Makefiles that use functions to match and transform sets of files. + +:::::::::::::::::::::::::::::::::::::::::::::::::: + +:::::::::::::::::::::::::::::::::::::::: questions + +- How *else* can I eliminate redundancy in my Makefiles? + +:::::::::::::::::::::::::::::::::::::::::::::::::: + At this point, we have the following Makefile: -~~~ +```make include config.mk # Generate summary table. @@ -32,51 +36,50 @@ dats : isles.dat abyss.dat last.dat clean : rm -f *.dat rm -f results.txt -~~~ -{: .language-make} +``` -Make has many [functions]({{ page.root }}/reference.html#function) which can be used +Make has many [functions](../learners/reference.md#function) which can be used to write more complex rules. One example is `wildcard`. `wildcard` gets a list of files matching some pattern, which we can then save in a variable. So, for example, we can get a list of all our text files (files ending in `.txt`) and save these in a variable by adding this at the beginning of our makefile: -~~~ +```make TXT_FILES=$(wildcard books/*.txt) -~~~ -{: .language-make} +``` We can add a `.PHONY` target and rule to show the variable's value: -~~~ +```make .PHONY : variables variables: @echo TXT_FILES: $(TXT_FILES) -~~~ -{: .language-make} +``` + +::::::::::::::::::::::::::::::::::::::::: callout -> ## @echo -> -> Make prints actions as it executes them. Using `@` at the start of -> an action tells Make not to print this action. So, by using `@echo` -> instead of `echo`, we can see the result of `echo` (the variable's -> value being printed) but not the `echo` command itself. -{: .callout} +## @echo + +Make prints actions as it executes them. Using `@` at the start of +an action tells Make not to print this action. So, by using `@echo` +instead of `echo`, we can see the result of `echo` (the variable's +value being printed) but not the `echo` command itself. + + +:::::::::::::::::::::::::::::::::::::::::::::::::: If we run Make: -~~~ +```bash $ make variables -~~~ -{: .language-bash} +``` We get: -~~~ +```output TXT_FILES: books/abyss.txt books/isles.txt books/last.txt books/sierra.txt -~~~ -{: .output} +``` Note how `sierra.txt` is now included too. @@ -87,41 +90,37 @@ variable. So, for example, we can rewrite our list of text files into a list of data files (files ending in `.dat`) and save these in a variable: -~~~ +```make DAT_FILES=$(patsubst books/%.txt, %.dat, $(TXT_FILES)) -~~~ -{: .language-make} +``` We can extend `variables` to show the value of `DAT_FILES` too: -~~~ +```make .PHONY : variables variables: @echo TXT_FILES: $(TXT_FILES) @echo DAT_FILES: $(DAT_FILES) -~~~ -{: .language-make} +``` If we run Make, -~~~ +```bash $ make variables -~~~ -{: .language-bash} +``` then we get: -~~~ +```output TXT_FILES: books/abyss.txt books/isles.txt books/last.txt books/sierra.txt DAT_FILES: abyss.dat isles.dat last.dat sierra.dat -~~~ -{: .output} +``` Now, `sierra.txt` is processed too. With these we can rewrite `clean` and `dats`: -~~~ +```make .PHONY : dats dats : $(DAT_FILES) @@ -129,77 +128,68 @@ dats : $(DAT_FILES) clean : rm -f $(DAT_FILES) rm -f results.txt -~~~ -{: .language-make} - +``` Let's check: -~~~ +```bash $ make clean $ make dats -~~~ -{: .language-bash} +``` We get: -~~~ +```output python countwords.py books/abyss.txt abyss.dat python countwords.py books/isles.txt isles.dat python countwords.py books/last.txt last.dat python countwords.py books/sierra.txt sierra.dat -~~~ -{: .output} +``` We can also rewrite `results.txt`: -~~~ +```make results.txt : $(ZIPF_SRC) $(DAT_FILES) $(LANGUAGE) $^ > $@ -~~~ -{: .language-make} +``` If we re-run Make: -~~~ +```bash $ make clean $ make results.txt -~~~ -{: .language-bash} +``` We get: -~~~ +```output python countwords.py books/abyss.txt abyss.dat python countwords.py books/isles.txt isles.dat python countwords.py books/last.txt last.dat python countwords.py books/sierra.txt sierra.dat python testzipf.py last.dat isles.dat abyss.dat sierra.dat > results.txt -~~~ -{: .output} +``` Let's check the `results.txt` file: -~~~ +```bash $ cat results.txt -~~~ -{: .language-bash} +``` -~~~ +```output Book First Second Ratio abyss 4044 2807 1.44 isles 3822 2460 1.55 last 12244 5566 2.20 sierra 4242 2469 1.72 -~~~ -{: .output} +``` So the range of the ratios of occurrences of the two most frequent words in our books is indeed around 2, as predicted by Zipf's Law, i.e., the most frequently-occurring word occurs approximately twice as often as the second most frequent word. Here is our final Makefile: -~~~ +```make include config.mk TXT_FILES=$(wildcard books/*.txt) @@ -225,49 +215,65 @@ clean : variables: @echo TXT_FILES: $(TXT_FILES) @echo DAT_FILES: $(DAT_FILES) -~~~ -{: .language-make} +``` Remember, the `config.mk` file contains: -~~~ +```make # Count words script. LANGUAGE=python COUNT_SRC=countwords.py # Test Zipf's rule ZIPF_SRC=testzipf.py -~~~ -{: .language-make} +``` The following figure shows the dependencies embodied within our Makefile, involved in building the `results.txt` target, now we have introduced our function: -![results.txt dependencies after introducing a function](../fig/07-functions.png "results.txt dependencies after introducing a function") - -> ## Where We Are -> -> [This Makefile]({{ page.root }}/code/07-functions/Makefile) -> and [its accompanying `config.mk`]({{ page.root }}/code/07-functions/config.mk) -> contain all of our work so far. -{: .callout} - -> ## Adding more books -> -> We can now do a better job at testing Zipf's rule by adding more books. -> The books we have used come from the [Project Gutenberg](http://www.gutenberg.org/) website. -> Project Gutenberg offers thousands of free ebooks to download. -> -> **Exercise instructions:** -> -> * go to [Project Gutenberg](http://www.gutenberg.org/) and use the search box to find another book, -> for example ['The Picture of Dorian Gray'](https://www.gutenberg.org/ebooks/174) from Oscar Wilde. -> * download the 'Plain Text UTF-8' version and save it to the `books` folder; -> choose a short name for the file (**that doesn't include spaces**) e.g. "dorian_gray.txt" -> because the filename is going to be used in the `results.txt` file -> * optionally, open the file in a text editor and remove extraneous text at the beginning and end -> (look for the phrase `END OF THE PROJECT GUTENBERG EBOOK [title]`) -> * run `make` and check that the correct commands are run, given the dependency tree -> * check the results.txt file to see how this book compares to the others -{: .challenge} +![](fig/07-functions.png "results.txt dependencies after introducing a function"){alt='results.txt dependencies after introducing a function'} + +::::::::::::::::::::::::::::::::::::::::: callout + +## Where We Are + +[This Makefile](code/07-functions/Makefile) +and [its accompanying `config.mk`](code/07-functions/config.mk) +contain all of our work so far. + + +:::::::::::::::::::::::::::::::::::::::::::::::::: + +::::::::::::::::::::::::::::::::::::::: challenge + +## Adding more books + +We can now do a better job at testing Zipf's rule by adding more books. +The books we have used come from the [Project Gutenberg](https://www.gutenberg.org/) website. +Project Gutenberg offers thousands of free ebooks to download. + +**Exercise instructions:** + +- go to [Project Gutenberg](https://www.gutenberg.org/) and use the search box to find another book, + for example ['The Picture of Dorian Gray'](https://www.gutenberg.org/ebooks/174) from Oscar Wilde. +- download the 'Plain Text UTF-8' version and save it to the `books` folder; + choose a short name for the file (**that doesn't include spaces**) e.g. "dorian\_gray.txt" + because the filename is going to be used in the `results.txt` file +- optionally, open the file in a text editor and remove extraneous text at the beginning and end + (look for the phrase `END OF THE PROJECT GUTENBERG EBOOK [title]`) +- run `make` and check that the correct commands are run, given the dependency tree +- check the results.txt file to see how this book compares to the others + + +:::::::::::::::::::::::::::::::::::::::::::::::::: + +:::::::::::::::::::::::::::::::::::::::: keypoints + +- Make is actually a small programming language with many built-in functions. +- Use `wildcard` function to get lists of files matching a pattern. +- Use `patsubst` function to rewrite file names. + +:::::::::::::::::::::::::::::::::::::::::::::::::: + + diff --git a/episodes/08-self-doc.md b/episodes/08-self-doc.md index af0d3ba1..3d3237c7 100644 --- a/episodes/08-self-doc.md +++ b/episodes/08-self-doc.md @@ -1,15 +1,21 @@ --- -title: "Self-Documenting Makefiles" +title: Self-Documenting Makefiles teaching: 10 exercises: 0 -questions: -- "How should I document a Makefile?" -objectives: -- "Write self-documenting Makefiles with built-in help." -keypoints: -- "Document Makefiles by adding specially-formatted comments and a target to extract and format them." --- +::::::::::::::::::::::::::::::::::::::: objectives + +- Write self-documenting Makefiles with built-in help. + +:::::::::::::::::::::::::::::::::::::::::::::::::: + +:::::::::::::::::::::::::::::::::::::::: questions + +- How should I document a Makefile? + +:::::::::::::::::::::::::::::::::::::::::::::::::: + Many bash commands, and programs that people have written that can be run from within bash, support a `--help` flag to display more information on how to use the commands or programs. In this spirit, it @@ -19,28 +25,25 @@ the key targets and what they do, so we don't need to look at the Makefile itself unless we want to. For our Makefile, running a `help` target might print: -~~~ +```bash $ make help -~~~ -{: .language-bash} +``` -~~~ +```output results.txt : Generate Zipf summary table. dats : Count words in text files. clean : Remove auto-generated files. -~~~ -{: .output} +``` So, how would we implement this? We could write a rule like: -~~~ +```make .PHONY : help help : @echo "results.txt : Generate Zipf summary table." @echo "dats : Count words in text files." @echo "clean : Remove auto-generated files." -~~~ -{: .language-make} +``` But every time we add or remove a rule, or change the description of a rule, we would have to update this rule too. It would be better if we @@ -56,7 +59,7 @@ which `sed` can detect. Since Make uses `#` for comments, we can use `##` for comments that describe what a rule does and that we want `sed` to detect. For example: -~~~ +```make ## results.txt : Generate Zipf summary table. results.txt : $(ZIPF_SRC) $(DAT_FILES) $(LANGUAGE) $^ > $@ @@ -79,8 +82,7 @@ clean : variables: @echo TXT_FILES: $(TXT_FILES) @echo DAT_FILES: $(DAT_FILES) -~~~ -{: .language-make} +``` We use `##` so we can distinguish between comments that we want `sed` to automatically filter, and other comments that may describe what @@ -88,12 +90,11 @@ other rules do, or that describe variables. We can then write a `help` target that applies `sed` to our `Makefile`: -~~~ +```make .PHONY : help help : Makefile @sed -n 's/^##//p' $< -~~~ -{: .language-make} +``` This rule depends upon the Makefile itself. It runs `sed` on the first dependency of the rule, which is our Makefile, and tells `sed` to get @@ -101,20 +102,18 @@ all the lines that begin with `##`, which `sed` then prints for us. If we now run -~~~ +```bash $ make help -~~~ -{: .language-bash} +``` we get: -~~~ +```output results.txt : Generate Zipf summary table. dats : Count words in text files. clean : Remove auto-generated files. variables : Print variables. -~~~ -{: .output} +``` If we add, change or remove a target or rule, we now only need to remember to add, update or remove a comment next to the rule. So long @@ -122,11 +121,24 @@ as we respect our convention of using `##` for such comments, then our `help` rule will take care of detecting these comments and printing them for us. -> ## Where We Are -> -> [This Makefile]({{ page.root }}/code/08-self-doc/Makefile) -> and [its accompanying `config.mk`]({{ page.root }}/code/08-self-doc/config.mk) -> contain all of our work so far. -{: .callout} +::::::::::::::::::::::::::::::::::::::::: callout + +## Where We Are + +[This Makefile](code/08-self-doc/Makefile) +and [its accompanying `config.mk`](code/08-self-doc/config.mk) +contain all of our work so far. + + +:::::::::::::::::::::::::::::::::::::::::::::::::: [sed-docs]: https://www.gnu.org/software/sed/ + + +:::::::::::::::::::::::::::::::::::::::: keypoints + +- Document Makefiles by adding specially-formatted comments and a target to extract and format them. + +:::::::::::::::::::::::::::::::::::::::::::::::::: + + diff --git a/episodes/09-conclusion.md b/episodes/09-conclusion.md index 5bdf4a83..bbb663e9 100644 --- a/episodes/09-conclusion.md +++ b/episodes/09-conclusion.md @@ -1,15 +1,21 @@ --- -title: "Conclusion" +title: Conclusion teaching: 5 exercises: 30 -questions: -- "What are the advantages and disadvantages of using tools like Make?" -objectives: -- "Understand advantages of automated build tools such as Make." -keypoints: -- "Makefiles save time by automating repetitive work, and save thinking by documenting how to reproduce results." --- +::::::::::::::::::::::::::::::::::::::: objectives + +- Understand advantages of automated build tools such as Make. + +:::::::::::::::::::::::::::::::::::::::::::::::::: + +:::::::::::::::::::::::::::::::::::::::: questions + +- What are the advantages and disadvantages of using tools like Make? + +:::::::::::::::::::::::::::::::::::::::::::::::::: + Automated build tools such as Make can help us in a number of ways. They help us to automate repetitive commands, hence saving us time and reducing the likelihood of errors compared with running @@ -24,109 +30,153 @@ as a form of documentation, recording dependencies between code, scripts, tools, configurations, raw data, derived data, plots, and papers. -> ## Creating PNGs -> -> Add new rules, update existing rules, and add new variables to: -> -> * Create `.png` files from `.dat` files using `plotcounts.py`. -> * Remove all auto-generated files (`.dat`, `.png`, -> `results.txt`). -> -> Finally, many Makefiles define a default [phony -> target]({{ page.root }}/reference.html#phony-target) called `all` as first target, -> that will build what the Makefile has been written to build (e.g. in -> our case, the `.png` files and the `results.txt` file). As others -> may assume your Makefile conforms to convention and supports an -> `all` target, add an `all` target to your Makefile (Hint: this rule -> has the `results.txt` file and the `.png` files as dependencies, but -> no actions). With that in place, instead of running `make -> results.txt`, you should now run `make all`, or just simply -> `make`. By default, `make` runs the first target it finds in the -> Makefile, in this case your new `all` target. -> -> > ## Solution -> > [This Makefile]({{ page.root }}/code/09-conclusion-challenge-1/Makefile) -> > and [this `config.mk`]({{ page.root }}/code/09-conclusion-challenge-1/config.mk) -> > contain a solution to this challenge. -> {: .solution} -{: .challenge} +::::::::::::::::::::::::::::::::::::::: challenge + +## Creating PNGs + +Add new rules, update existing rules, and add new variables to: + +- Create `.png` files from `.dat` files using `plotcounts.py`. +- Remove all auto-generated files (`.dat`, `.png`, + `results.txt`). + +Finally, many Makefiles define a default [phony +target](../learners/reference.md#phony-target) called `all` as first target, +that will build what the Makefile has been written to build (e.g. in +our case, the `.png` files and the `results.txt` file). As others +may assume your Makefile conforms to convention and supports an +`all` target, add an `all` target to your Makefile (Hint: this rule +has the `results.txt` file and the `.png` files as dependencies, but +no actions). With that in place, instead of running `make results.txt`, you should now run `make all`, or just simply +`make`. By default, `make` runs the first target it finds in the +Makefile, in this case your new `all` target. + +::::::::::::::: solution + +## Solution + +[This Makefile](code/09-conclusion-challenge-1/Makefile) +and [this `config.mk`](code/09-conclusion-challenge-1/config.mk) +contain a solution to this challenge. + + + +::::::::::::::::::::::::: + +:::::::::::::::::::::::::::::::::::::::::::::::::: The following figure shows the dependencies involved in building the `all` target, once we've added support for images: -![results.txt dependencies once images have been added](../fig/09-conclusion-challenge-1.png "results.txt dependencies once images have been added") - -> ## Creating an Archive -> -> Often it is useful to create an archive file of your project that includes all data, code -> and results. An archive file can package many files into a single file that can easily be -> downloaded and shared with collaborators. We can add steps to create the archive file inside -> the Makefile itself so it's easy to update our archive file as the project changes. -> -> -> Edit the Makefile to create an archive file of your project. Add new rules, update existing -> rules and add new variables to: -> -> * Create a new directory called `zipf_analysis` in the project directory. -> * Copy all our code, data, plots, the Zipf summary table, the Makefile and config.mk -> to this directory. -> The `cp -r` command can be used to copy files and directories -> into the new `zipf_analysis` directory: -> -> ~~~ -> $ cp -r [files and directories to copy] zipf_analysis/ -> ~~~ -> {: .language-bash} -> -> * Hint: create a new variable for the `books` directory so that it can be -> copied to the new `zipf_analysis` directory -> * Create an archive, `zipf_analysis.tar.gz`, of this directory. The -> bash command `tar` can be used, as follows: -> -> ~~~ -> $ tar -czf zipf_analysis.tar.gz zipf_analysis -> ~~~ -> {: .language-bash} -> -> * Update the target `all` so that it creates `zipf_analysis.tar.gz`. -> * Remove `zipf_analysis.tar.gz` when `make clean` is called. -> * Print the values of any additional variables you have defined when -> `make variables` is called. -> -> > ## Solution -> > [This Makefile]({{ page.root }}/code/09-conclusion-challenge-2/Makefile) -> > and [this `config.mk`]({{ page.root }}/code/09-conclusion-challenge-2/config.mk) -> > contain a solution to this challenge. -> {: .solution} -{: .challenge} - -> ## Archiving the Makefile -> -> Why does the Makefile rule for the archive directory add the Makefile to our archive of code, -> data, plots and Zipf summary table? -> -> > ## Solution -> > Our code files (`countwords.py`, `plotcounts.py`, `testzipf.py`) implement -> > the individual parts of our workflow. They allow us to create `.dat` -> > files from `.txt` files, and `results.txt` and `.png` files from `.dat` files. -> > Our Makefile, however, documents dependencies between -> > our code, raw data, derived data, and plots, as well as implementing -> > our workflow as a whole. `config.mk` contains configuration information -> > for our Makefile, so it must be archived too. -> {: .solution} -{: .challenge} - -> ## `touch` the Archive Directory -> -> Why does the Makefile rule for the archive directory `touch` the archive directory after moving our code, data, plots and summary table into it? -> -> > ## Solution -> > A directory's timestamp is not automatically updated when files are copied into it. -> > If the code, data, plots, and summary table are updated and copied into the -> > archive directory, the archive directory's timestamp must be updated with `touch` -> > so that the rule that makes `zipf_analysis.tar.gz` knows to run again; -> > without this `touch`, `zipf_analysis.tar.gz` will only be created the first time -> > the rule is run and will not be updated on subsequent runs even if the contents -> > of the archive directory have changed. -> {: .solution} -{: .challenge} +![](fig/09-conclusion-challenge-1.png "results.txt dependencies once images have been added"){alt='results.txt dependencies once images have been added'} + +::::::::::::::::::::::::::::::::::::::: challenge + +## Creating an Archive + +Often it is useful to create an archive file of your project that includes all data, code +and results. An archive file can package many files into a single file that can easily be +downloaded and shared with collaborators. We can add steps to create the archive file inside +the Makefile itself so it's easy to update our archive file as the project changes. + +Edit the Makefile to create an archive file of your project. Add new rules, update existing +rules and add new variables to: + +- Create a new directory called `zipf_analysis` in the project directory. + +- Copy all our code, data, plots, the Zipf summary table, the Makefile and config.mk + to this directory. + The `cp -r` command can be used to copy files and directories + into the new `zipf_analysis` directory: + + ```bash + $ cp -r [files and directories to copy] zipf_analysis/ + ``` + +- Hint: create a new variable for the `books` directory so that it can be + copied to the new `zipf_analysis` directory + +- Create an archive, `zipf_analysis.tar.gz`, of this directory. The + bash command `tar` can be used, as follows: + + ```bash + $ tar -czf zipf_analysis.tar.gz zipf_analysis + ``` + +- Update the target `all` so that it creates `zipf_analysis.tar.gz`. + +- Remove `zipf_analysis.tar.gz` when `make clean` is called. + +- Print the values of any additional variables you have defined when + `make variables` is called. + +::::::::::::::: solution + +## Solution + +[This Makefile](code/09-conclusion-challenge-2/Makefile) +and [this `config.mk`](code/09-conclusion-challenge-2/config.mk) +contain a solution to this challenge. + + + +::::::::::::::::::::::::: + +:::::::::::::::::::::::::::::::::::::::::::::::::: + +::::::::::::::::::::::::::::::::::::::: challenge + +## Archiving the Makefile + +Why does the Makefile rule for the archive directory add the Makefile to our archive of code, +data, plots and Zipf summary table? + +::::::::::::::: solution + +## Solution + +Our code files (`countwords.py`, `plotcounts.py`, `testzipf.py`) implement +the individual parts of our workflow. They allow us to create `.dat` +files from `.txt` files, and `results.txt` and `.png` files from `.dat` files. +Our Makefile, however, documents dependencies between +our code, raw data, derived data, and plots, as well as implementing +our workflow as a whole. `config.mk` contains configuration information +for our Makefile, so it must be archived too. + + + +::::::::::::::::::::::::: + +:::::::::::::::::::::::::::::::::::::::::::::::::: + +::::::::::::::::::::::::::::::::::::::: challenge + +## `touch` the Archive Directory + +Why does the Makefile rule for the archive directory `touch` the archive directory after moving our code, data, plots and summary table into it? + +::::::::::::::: solution + +## Solution + +A directory's timestamp is not automatically updated when files are copied into it. +If the code, data, plots, and summary table are updated and copied into the +archive directory, the archive directory's timestamp must be updated with `touch` +so that the rule that makes `zipf_analysis.tar.gz` knows to run again; +without this `touch`, `zipf_analysis.tar.gz` will only be created the first time +the rule is run and will not be updated on subsequent runs even if the contents +of the archive directory have changed. + + + +::::::::::::::::::::::::: + +:::::::::::::::::::::::::::::::::::::::::::::::::: + +:::::::::::::::::::::::::::::::::::::::: keypoints + +- Makefiles save time by automating repetitive work, and save thinking by documenting how to reproduce results. + +:::::::::::::::::::::::::::::::::::::::::::::::::: + + diff --git a/data/books/LICENSE_TEXTS.md b/episodes/data/books/LICENSE_TEXTS.md similarity index 100% rename from data/books/LICENSE_TEXTS.md rename to episodes/data/books/LICENSE_TEXTS.md diff --git a/data/books/abyss.txt b/episodes/data/books/abyss.txt similarity index 100% rename from data/books/abyss.txt rename to episodes/data/books/abyss.txt diff --git a/data/books/isles.txt b/episodes/data/books/isles.txt similarity index 100% rename from data/books/isles.txt rename to episodes/data/books/isles.txt diff --git a/data/books/last.txt b/episodes/data/books/last.txt similarity index 100% rename from data/books/last.txt rename to episodes/data/books/last.txt diff --git a/data/books/sierra.txt b/episodes/data/books/sierra.txt similarity index 100% rename from data/books/sierra.txt rename to episodes/data/books/sierra.txt diff --git a/fig/02-makefile-challenge.png b/episodes/fig/02-makefile-challenge.png similarity index 100% rename from fig/02-makefile-challenge.png rename to episodes/fig/02-makefile-challenge.png diff --git a/fig/02-makefile.png b/episodes/fig/02-makefile.png similarity index 100% rename from fig/02-makefile.png rename to episodes/fig/02-makefile.png diff --git a/fig/04-dependencies.png b/episodes/fig/04-dependencies.png similarity index 100% rename from fig/04-dependencies.png rename to episodes/fig/04-dependencies.png diff --git a/fig/07-functions.png b/episodes/fig/07-functions.png similarity index 100% rename from fig/07-functions.png rename to episodes/fig/07-functions.png diff --git a/fig/09-conclusion-challenge-1.png b/episodes/fig/09-conclusion-challenge-1.png similarity index 100% rename from fig/09-conclusion-challenge-1.png rename to episodes/fig/09-conclusion-challenge-1.png diff --git a/files/make-lesson.zip b/episodes/files/make-lesson.zip similarity index 100% rename from files/make-lesson.zip rename to episodes/files/make-lesson.zip diff --git a/index.md b/index.md index 771472a2..a6728531 100644 --- a/index.md +++ b/index.md @@ -1,7 +1,6 @@ --- -layout: lesson -root: . permalink: index.html +site: sandpaper::sandpaper_site --- Make is a tool which can run commands to read files, process these @@ -10,10 +9,10 @@ in software development, Make is used to compile source code into executable programs or libraries, but Make can also be used to: -* run analysis scripts on raw data files to get data files that +- run analysis scripts on raw data files to get data files that summarize the raw data; -* run visualization scripts on data files to produce plots; and to -* parse and combine text files and plots to create papers. +- run visualization scripts on data files to produce plots; and to +- parse and combine text files and plots to create papers. Make is called a build tool - it builds data files, plots, papers, programs or libraries. It can also update existing files if @@ -27,15 +26,26 @@ that depend upon this file (e.g. a plot). There are now many build tools available, all of which are based on the same concepts as Make. -> ## Prerequisites -> -> In this lesson we use `make` from the Unix Shell. Some previous -> experience with using the shell to list directories, create, copy, -> remove and list files and directories, and run simple scripts is -> necessary. -{: .prereq} - -> ## Setup -> In order to follow this lesson, you will need to download some files. -> Please follow instructions on the [setup]({{ page.root }}{% link setup.md %}) page. -{: .prereq} +:::::::::::::::::::::::::::::::::::::::::: prereq + +## Prerequisites + +In this lesson we use `make` from the Unix Shell. Some previous +experience with using the shell to list directories, create, copy, +remove and list files and directories, and run simple scripts is +necessary. + + +:::::::::::::::::::::::::::::::::::::::::::::::::: + +:::::::::::::::::::::::::::::::::::::::::: prereq + +## Setup + +In order to follow this lesson, you will need to download some files. +Please follow instructions on the [setup](learners/setup.md) page. + + +:::::::::::::::::::::::::::::::::::::::::::::::::: + + diff --git a/_extras/guide.md b/instructors/instructor-notes.md similarity index 85% rename from _extras/guide.md rename to instructors/instructor-notes.md index ac4bc2f0..e27ac627 100644 --- a/_extras/guide.md +++ b/instructors/instructor-notes.md @@ -38,10 +38,10 @@ that the text in both windows is readable from the back of the room. Recommend instructors and students use `nano` as the text editor for this lesson because -* it runs in all three major operating systems, -* it runs inside the shell (switching windows can be confusing to +- it runs in all three major operating systems, +- it runs inside the shell (switching windows can be confusing to students), and -* it has shortcut help at the bottom of the window. +- it has shortcut help at the bottom of the window. Please point out to students during setup that they can and should use another text editor if they're already familiar with it. @@ -61,20 +61,19 @@ Data files are in `data/books`. You can either create a simple Git repository for students to clone which contains: -* `countwords.py` -* `plotcounts.py` -* `testzipf.py` -* `books/` +- `countwords.py` +- `plotcounts.py` +- `testzipf.py` +- `books/` Or, ask students to download [make-lesson.zip][zipfile] from this repository. To recreate `make-lesson.zip`, run: -~~~ +```bash $ make make-lesson.zip -~~~ -{: .language-bash} +``` ## Beware of Spaces! @@ -85,37 +84,35 @@ spaces instead of TABs when indenting actions. Some of these pages use images of Makefile dependencies, in the `fig` directory. -These are created using [makefile2graph][makefile2graph], +These are created using [makefile2graph], which is assumed to be in the `PATH`. This tool, in turn, needs the `dot` tool, part of [GraphViz][graphviz]. To install GraphViz on Scientific Linux 6: -~~~ +```bash $ sudo yum install graphviz $ dot -V -~~~ -{: .language-bash} -~~~ +``` + +```output dot - graphviz version 2.26.0 (20091210.2329) -~~~ -{: .output} +``` To install GraphViz on Ubuntu: -~~~ +```bash $ sudo apt-get install graphviz $ dot -V -~~~ -{: .language-bash} -~~~ +``` + +```output dot - graphviz version 2.38.0 (20140413.2041) -~~~ -{: .output} +``` To download and build makefile2graph on Linux: -~~~ +```bash $ cd $ git clone https://github.com/lindenb/makefile2graph $ cd makefile2graph/ @@ -123,19 +120,17 @@ $ make $ export PATH=~/makefile2graph/:$PATH $ cd $ which makefile2graph -~~~ -{: .language-bash} -~~~ +``` + +```output /home/ubuntu/makefile2graph/makefile2graph -~~~ -{: .output} +``` To create the image files for the lesson: -~~~ +```bash $ make diagrams -~~~ -{: .language-bash} +``` See `commands.mk`'s `diagrams` target. @@ -144,11 +139,11 @@ See `commands.mk`'s `diagrams` target. When processing `books/last.txt` with Python 3 and vanilla shell environment on Arch Linux the following error has appeared: -~~~ +```bash $ python wordcount.py books/last.txt last.dat -~~~ -{: .language-bash} -~~~ +``` + +```output Traceback (most recent call last): File "wordcount.py", line 131, in word_count(input_file, output_file, min_length) @@ -159,18 +154,16 @@ Traceback (most recent call last): File "/usr/lib/python3.6/encodings/ascii.py", line 26, in decode return codecs.ascii_decode(input, self.errors)[0] UnicodeDecodeError: 'ascii' codec can't decode byte 0xc3 in position 6862: ordinal not in range(128) -~~~ -{: .output} +``` The workaround was to define encoding for the terminal session (this can be either done at the command line or placed in the `.bashrc` or equivalent): -~~~ +```bash $ export LC_ALL=en_US.UTF-8 $ export LANG=en_US.UTF-8 $ export LANGUAGE=en_US.UTF-8 -~~~ -{: .language-bash} +``` ## Beware of different Make implementations! @@ -178,7 +171,9 @@ The lesson is based on GNU Make. Although it is very rare, on some systems (e.g. AIX) you might find `make` not pointing to GNU Make and `gmake` needs to be used instead. -[graphviz]: http://www.graphviz.org/ -[lesson-example]: https://github.com/carpentries/lesson-example/ +[zipfile]: files/make-lesson.zip [makefile2graph]: https://github.com/lindenb/makefile2graph -[zipfile]: {{ page.root }}/files/make-lesson.zip +[graphviz]: https://www.graphviz.org/ + + + diff --git a/_extras/discuss.md b/learners/discuss.md similarity index 66% rename from _extras/discuss.md rename to learners/discuss.md index c7cde313..f9447ce1 100644 --- a/_extras/discuss.md +++ b/learners/discuss.md @@ -4,14 +4,13 @@ title: Discussion ## Parallel Execution -Make can build dependencies in _parallel_ sub-processes, via its `--jobs` +Make can build dependencies in *parallel* sub-processes, via its `--jobs` flag (or its `-j` abbreviation) which specifies the number of sub-processes to use e.g. -~~~ +```bash $ make --jobs 4 results.txt -~~~ -{: .language-bash} +``` If we have independent dependencies then these can be built at the same time. For example, `abyss.dat` and `isles.dat` are mutually @@ -28,25 +27,25 @@ Execution][gnu-make-parallel]. Some Makefiles may contain `:=` instead of `=`. Your Makefile may behave differently depending upon which you use and how you use it: -* A variable defined using `=` is a _recursively expanded - variable_. Its value is calculated only when its value is +- A variable defined using `=` is a *recursively expanded + variable*. Its value is calculated only when its value is requested. If the value assigned to the variable itself contains variables (e.g. `A = $(B)`) then these variables' values are only calculated when the variable's value is requested (e.g. the value of `B` is only calculated when the value of `A` is requested via - `$(A)`. This can be termed _lazy setting_. + `$(A)`. This can be termed *lazy setting*. -* A variable defined using `:=` is a _simply expanded variable_. Its +- A variable defined using `:=` is a *simply expanded variable*. Its value is calculated when it is declared. If the value assigned to the variable contains variables (e.g. `A = $(B)`) then these variables' values are also calculated when the variable is declared (e.g. the value of `B` is calculated when `A` is assigned - above). This can be termed _immediate setting_. + above). This can be termed *immediate setting*. For a detailed explanation, see: -* StackOverflow [Makefile variable assignment][makefile-variable] -* GNU Make [The Two Flavors of Variables][gnu-make-variables] +- StackOverflow [Makefile variable assignment][makefile-variable] +- GNU Make [The Two Flavors of Variables][gnu-make-variables] ## Make and Version Control @@ -83,21 +82,21 @@ executed to update an object. More complex actions could well include shell variables. There are several ways in which make variables and shell variables can be confused and can be in conflict. -* Make actually accepts three different syntaxes for variables: `$N`, +- Make actually accepts three different syntaxes for variables: `$N`, `$(NAME)`, or `${NAME}`. - + The single character variable names are most commonly used for automatic variables, and there are many of them. But if you happen upon a character that isn't pre-defined as an automatic variable, make will treat it as a user variable. - + The `${NAME}` syntax is also used by the unix shell in cases where there might be ambiguity in interpreting variable names, or for certain pattern substitution operations. Since there are only certain situations in which the unix shell requires this syntax, instead of the more common `$NAME`, it is not familiar to many users. -* Make does variable substitution on actions before they are passed to +- Make does variable substitution on actions before they are passed to the shell for execution. That means that anything that looks like a variable to make will get replaced with the appropriate value. (In make, an uninitialized variable has a null value.) To protect a @@ -112,109 +111,112 @@ shell variables can be confused and can be in conflict. instead of `awk '{print $1}'`) or accessing environment variables (e.g., `$$HOME`). -> ## Detailed Example of Shell Variable Quoting -> -> Say we had the following `Makefile` (and the .dat files had already -> been created): -> -> ~~~ -> BOOKS = abyss isles -> -> .PHONY: plots -> plots: -> for book in $(BOOKS); do python plotcount.py $book.dat $book.png; done -> ~~~ -> {: .language-make} -> -> the action that would be passed to the shell to execute would be: -> -> ~~~ -> for book in abyss isles; do python plotcount.py ook.dat ook.png; done -> ~~~ -> {: .language-bash} -> -> Notice that make substituted `$(BOOKS)`, as expected, but it also -> substituted `$book`, even though we intended it to be a shell variable. -> Moreover, because we didn't use `$(NAME)` (or `${NAME}`) syntax, make -> interpreted it as the single character variable `$b` (which we haven't -> defined, so it has a null value) followed by the text "ook". -> -> In order to get the desired behavior, we have to write `$$book` instead -> of `$book`: -> -> ~~~ -> BOOKS = abyss isles -> -> .PHONY: plots -> plots: -> for book in $(BOOKS); do python plotcount.py $$book.dat $$book.png; done -> ~~~ -> {: .language-make} -> -> which produces the correct shell command: -> -> ~~~ -> for book in abyss isles; do python plotcount.py $book.dat $book.png; done -> ~~~ -> {: .language-bash} -{: .discussion} +:::::::::::::::::::::::::::::::::::::: discussion + +## Detailed Example of Shell Variable Quoting + +Say we had the following `Makefile` (and the .dat files had already +been created): + +```make +BOOKS = abyss isles + +.PHONY: plots +plots: + for book in $(BOOKS); do python plotcount.py $book.dat $book.png; done +``` + +the action that would be passed to the shell to execute would be: + +```bash +for book in abyss isles; do python plotcount.py ook.dat ook.png; done +``` + +Notice that make substituted `$(BOOKS)`, as expected, but it also +substituted `$book`, even though we intended it to be a shell variable. +Moreover, because we didn't use `$(NAME)` (or `${NAME}`) syntax, make +interpreted it as the single character variable `$b` (which we haven't +defined, so it has a null value) followed by the text "ook". + +In order to get the desired behavior, we have to write `$$book` instead +of `$book`: + +```make +BOOKS = abyss isles + +.PHONY: plots +plots: + for book in $(BOOKS); do python plotcount.py $$book.dat $$book.png; done +``` + +which produces the correct shell command: + +```bash +for book in abyss isles; do python plotcount.py $book.dat $book.png; done +``` + +:::::::::::::::::::::::::::::::::::::::::::::::::: ## Make and Reproducible Research Blog articles, papers, and tutorials on automating commonly occurring research activities using Make: -* [minimal make][minimal-make] by Karl Broman. A minimal tutorial on +- [minimal make][minimal-make] by Karl Broman. A minimal tutorial on using Make with R and LaTeX to automate data analysis, visualization and paper preparation. This page has links to Makefiles for many of his papers. -* [Why Use Make][why-use-make] by Mike Bostock. An example of using +- [Why Use Make][why-use-make] by Mike Bostock. An example of using Make to download and convert data. -* [Makefiles for R/LaTeX projects][makefiles-for-r-latex] by Rob +- [Makefiles for R/LaTeX projects][makefiles-for-r-latex] by Rob Hyndman. Another example of using Make with R and LaTeX. -* [GNU Make for Reproducible Data Analysis][make-reproducible-research] +- [GNU Make for Reproducible Data Analysis][make-reproducible-research] by Zachary Jones. Using Make with Python and LaTeX. -* Shaun Jackman's [Using Make to Increase Automation & +- Shaun Jackman's [Using Make to Increase Automation \& Reproducibility][increase-automation] video lesson, and accompanying [example][increase-automation-example]. -* Lars Yencken's [Driving experiments with +- Lars Yencken's [Driving experiments with make][driving-experiments]. Using Make to sandbox Python dependencies and pull down data sets from Amazon S3. -* Askren MK, McAllister-Day TK, Koh N, Mestre Z, Dines JN, Korman BA, +- Askren MK, McAllister-Day TK, Koh N, Mestre Z, Dines JN, Korman BA, Melhorn SJ, Peterson DJ, Peverill M, Qin X, Rane SD, Reilly MA, Reiter MA, Sambrook KA, Woelfer KA, Grabowski TJ and Madhyastha TM (2016) [Using Make for Reproducible and Parallel Neuroimaging Workflow and Quality-Assurance][make-neuroscience]. Front. Neuroinform. 10:2. doi: - 10.3389/fninf.2016.00002 + 10\.3389/fninf.2016.00002 -* Li Haoyi's [What's in a Build Tool?][whats-a-build-tool] A review of +- Li Haoyi's [What's in a Build Tool?][whats-a-build-tool] A review of popular build tools (including Make) in terms of their strengths and weaknesses for common build-related use cases in software development. -[driving-experiments]: http://lifesum.github.io/posts/2016/01/14/make-experiments/ +## Return messages and `.PHONY` target behaviour + +`Up to date` vs `Nothing to be done` is discussed in +[episode 2](../episodes/02-makefiles.md). + +A more detailed discussion can be read on +[issue 98](https://github.com/swcarpentry/make-novice/issues/98#issuecomment-307361751). + [gnu-make-parallel]: https://www.gnu.org/software/make/manual/html_node/Parallel.html +[makefile-variable]: https://stackoverflow.com/questions/448910/makefile-variable-assignment [gnu-make-variables]: https://www.gnu.org/software/make/manual/html_node/Flavors.html#Flavors +[minimal-make]: https://kbroman.org/minimal_make/ +[why-use-make]: https://bost.ocks.org/mike/make/ +[makefiles-for-r-latex]: https://robjhyndman.com/hyndsight/makefiles/ +[make-reproducible-research]: https://zmjones.com/make/ [increase-automation]: https://www.youtube.com/watch?v=_F5f0qi-aEc [increase-automation-example]: https://github.com/sjackman/makefile-example -[make-neuroscience]: http://journal.frontiersin.org/article/10.3389/fninf.2016.00002/full -[make-reproducible-research]: http://zmjones.com/make/ -[makefile-variable]: http://stackoverflow.com/questions/448910/makefile-variable-assignment -[makefiles-for-r-latex]: http://robjhyndman.com/hyndsight/makefiles/ -[minimal-make]: http://kbroman.org/minimal_make/ -[whats-a-build-tool]: http://www.lihaoyi.com/post/WhatsinaBuildTool.html -[why-use-make]: http://bost.ocks.org/mike/make/ +[driving-experiments]: https://lifesum.github.io/posts/2016/01/14/make-experiments/ +[make-neuroscience]: https://journal.frontiersin.org/article/10.3389/fninf.2016.00002/full +[whats-a-build-tool]: https://www.lihaoyi.com/post/WhatsinaBuildTool.html + -## Return messages and `.PHONY` target behaviour -`Up to date` vs `Nothing to be done` is discussed in -[episode 2]({{page.root}}/02-makefiles/). -A more detailed discussion can be read on -[issue 98](https://github.com/swcarpentry/make-novice/issues/98#issuecomment-307361751). diff --git a/learners/reference.md b/learners/reference.md new file mode 100644 index 00000000..629bf9b4 --- /dev/null +++ b/learners/reference.md @@ -0,0 +1,345 @@ +--- +title: 'FIXME' +--- + +## Glossary + +## Running Make + +To run Make: + +```bash +$ make +``` + +Make will look for a Makefile called `Makefile` and will build the +default target, the first target in the Makefile. + +To use a Makefile with a different name, use the `-f` flag e.g. + +```bash +$ make -f build-files/analyze.mk +``` + +To build a specific target, provide it as an argument e.g. + +```bash +$ make isles.dat +``` + +If the target is up-to-date, Make will print a message like: + +```output +make: `isles.dat' is up to date. +``` + +To see the actions Make will run when building a target, without +running the actions, use the `--dry-run` flag e.g. + +```bash +$ make --dry-run isles.dat +``` + +Alternatively, use the abbreviation `-n`. + +```bash +$ make -n isles.dat +``` + +## Trouble Shooting + +If Make prints a message like, + +```error +Makefile:3: *** missing separator. Stop. +``` + +then check that all the actions are indented by TAB characters and not +spaces. + +If Make prints a message like, + +```error +No such file or directory: 'books/%.txt' +make: *** [isles.dat] Error 1 +``` + +then you may have used the Make wildcard, `%`, in an action in a +pattern rule. Make wildcards cannot be used in actions. + +## Makefiles + +Rules: + +```make +target : dependency1 dependency2 ... + action1 + action2 + ... +``` + +- Each rule has a target, a file to be created, or built. +- Each rule has zero or more dependencies, files that are needed to + build the target. +- `:` separates the target and the dependencies. +- Dependencies are separated by spaces. +- Each rule has zero or more actions, commands to run to build the + target using the dependencies. +- Actions are indented using the TAB character, not 8 spaces. + +Dependencies: + +- If any dependency does not exist then Make will look for a rule to + build it. +- The order of rebuilding dependencies is arbitrary. You should not + assume that they will be built in the order in which they are listed. +- Dependencies must form a directed acyclic graph. A target cannot + depend on a dependency which, in turn depends upon, or has a + dependency which depends upon, that target. + +Comments: + +```make +# This is a Make comment. +``` + +Line continuation character: + +```make +ARCHIVE = isles.dat isles.png \ + abyss.dat abyss.png \ + sierra.dat sierra.png +``` + +- If a list of dependencies or an action is too long, a Makefile can + become more difficult to read. +- Backslash,`\`, the line continuation character, allows you to split + up a list of dependencies or an action over multiple lines, to make + them easier to read. +- Make will combine the multiple lines into a single list of dependencies + or action. + +Phony targets: + +```make +.PHONY : clean +clean : + rm -f *.dat +``` + +- Phony targets are a short-hand for sequences of actions. +- No file with the target name is built when a rule with a phony + target is run. + +Automatic variables: + +- `$<` denotes 'the first dependency of the current rule'. +- `$@` denotes 'the target of the current rule'. +- `$^` denotes 'the dependencies of the current rule'. +- `$*` denotes 'the stem with which the pattern of the current rule matched'. + +Pattern rules: + +```make +%.dat : books/%.txt $(COUNT_SRC) + $(COUNT_EXE) $< $@ +``` + +- The Make wildcard, `%`, specifies a pattern. +- If Make finds a dependency matching the pattern, then the pattern is + substituted into the target. +- The Make wildcard can only be used in targets and dependencies. +- e.g. if Make found a file called `books/abyss.txt`, it would set the + target to be `abyss.dat`. + +Defining and using variables: + +```make +COUNT_SRC=wordcount.py +COUNT_EXE=python $(COUNT_SRC) +``` + +- A variable is assigned a value. For example, `COUNT_SRC` + is assigned the value `wordcount.py`. +- `$(...)` is a reference to a variable. It requests that + Make substitutes the name of a variable for its value. + +Suppress printing of actions: + +```make +.PHONY : variables +variables: + @echo TXT_FILES: $(TXT_FILES) +``` + +- Prefix an action by `@` to instruct Make not to print that action. + +Include the contents of a Makefile in another Makefile: + +```make +include config.mk +``` + +wildcard function: + +```make +TXT_FILES=$(wildcard books/*.txt) +``` + +- Looks for all files matching a pattern e.g. `books/*.txt`, and + return these in a list. +- e.g. `TXT_FILES` is set to `books/abyss.txt books/isles.txt books/last.txt books/sierra.txt`. + +patsubst ('path substitution') function: + +```make +DAT_FILES=$(patsubst books/%.txt, %.dat, $(TXT_FILES)) +``` + +- Every string that matches `books/%.txt` in `$(TXT_FILES)` is + replaced by `%.dat` and the strings are returned in a list. +- e.g. if `TXT_FILES` is `books/abyss.txt books/isles.txt books/last.txt books/sierra.txt` this sets `DAT_FILES` to `abyss.dat isles.dat last.dat sierra.dat`. + +Default targets: + +- In Make version 3.79 the default target is the first target in the + Makefile. +- In Make 3.81, the default target can be explicitly set using the + special variable `.DEFAULT_GOAL` e.g. + +```make +.DEFAULT_GOAL := all +``` + +## Manuals + +[GNU Make Manual][gnu-make-manual]. Reference sections include: + +- [Summary of Options][options-summary] for the `make` command. +- [Quick Reference][quick-reference] of Make directives, text manipulation functions, and special variables. +- [Automatic Variables][automatic-variables]. +- [Special Built-in Target Names][special-targets] + +## Glossary + +{:auto\_ids} +action +: The steps a [build manager](#build-manager) must take to create or +update a file or other object. + +assignment +: A request that [Make](#make) stores something in a +[variable](#variable). + +automatic variable +: A variable whose value is automatically redefined for each +[rule](#rule). [Make](#make)'s automatic variables include `$@`, +which holds the rule's [target](#target), `$^`, which holds its +[dependencies](#dependency), and, `$<`, which holds the first of +its dependencies, and `$*`, which holds the [stem](#stem) with which +the pattern was matched. Automatic variables are typically used in +[pattern rules](#pattern-rule). + +build file +: A description of [dependencies](#dependency) and [rules](#rule) +for a [build manager](#build-manager). + +build manager +: A program, such as [Make](#make), whose main purpose is to build or +update software, documentation, web sites, data files, images, and +other things. + +default rule +: The [rule](#rule) that is executed if no [target](#target) is +specified when a [build manager](#build-manager) is run. + +default target +: The [target](#target) of the [default rule](#default-rule). + +dependency +: A file that a [target](#target) depends on. If any of a target's +[dependencies](#dependency) are newer than the target itself, the +target needs to be updated. A target's dependencies are also +called its prerequisites. If a target's dependencies do not exist, +then they need to be built first. + +false dependency +: This can refer to a [dependency](#dependency) that is artificial. +e.g. a false dependency is introduced if a data analysis script +is added as a dependency to the data files that the script +analyses. + +function +: A built-in [Make](#make) utility that performs some operation, for +example gets a list of files matching a pattern. + +incremental build +: The feature of a [build manager](#build-manager) by +which it only rebuilds files that, either directory +or indirectly, depend on a file that was changed. + +macro +: Used as a synonym for [variable](#variable) in certain versions of +[Make](#make). + +Make +: A popular [build manager](#build-manager), from GNU, created in 1977. + +Makefile +: A [build file](#build-file) used by [Make](#make), which, by +default, are named `Makefile`. + +pattern rule +: A [rule](#rule) that specifies a general way to build or update an +entire class of files that can be managed the same way. For +example, a pattern rule can specify how to compile any C file +rather than a single, specific C file, or, to analyze any data +file rather than a single, specific data file. Pattern rules +typically make use of [automatic variables](#automatic-variable) +and [wildcards](#wildcard). + +phony target +: A [target](#target) that does not correspond to a file or other +object. Phony targets are usually symbolic names for sequences of +[actions](#action). + +prerequisite +: A synonym for [dependency](#dependency). + +reference +: A request that [Make](#make) substitutes the name of a +[variable](#variable) for its value. + +rule +: A specification of a [target](#target)'s +[dependencies](#dependency) and what [actions](#action) need to be +executed to build or update the target. + +stem +: The part of the target that was matched by the pattern rule. If +the target is `file.dat` and the target pattern was `%.dat`, then +the stem `$*` is `file`. + +target +: A thing to be created or updated, for example a file. Targets can +have [dependencies](#dependency) that must exist, and be +up-to-date, before the target itself can be built or updated. + +variable +: A symbolic name for something in a [Makefile](#makefile). + +wildcard +: A pattern that can be specified in [dependencies](#dependency) and +[targets](#target). If [Make](#make) finds a dependency matching +the pattern, then the pattern is substituted into the +target. wildcards are often used in [pattern +rules](#pattern-rule). The Make wildcard is `%`. + +[gnu-make-manual]: https://www.gnu.org/software/make/manual/ +[options-summary]: https://www.gnu.org/software/make/manual/html_node/Options-Summary.html +[quick-reference]: https://www.gnu.org/software/make/manual/html_node/Quick-Reference.html +[automatic-variables]: https://www.gnu.org/software/make/manual/html_node/Automatic-Variables.html +[special-targets]: https://www.gnu.org/software/make/manual/html_node/Special-Targets.html + + + diff --git a/setup.md b/learners/setup.md similarity index 89% rename from setup.md rename to learners/setup.md index 53a7eff7..bc73a2a1 100644 --- a/setup.md +++ b/learners/setup.md @@ -15,18 +15,16 @@ You need to download some files to follow this lesson: 4. Navigate to the directory where you downloaded the file. 5. Unpack `make-lesson.zip`: - - ~~~ - $ unzip make-lesson.zip - ~~~ - {: .source} + + ```source + $ unzip make-lesson.zip + ``` 6. Change into the `make-lesson` directory: - - ~~~ - $ cd make-lesson - ~~~ - {: .source} + + ```source + $ cd make-lesson + ``` ## Software @@ -50,6 +48,7 @@ You will need to have Xcode installed (download from the Check if you already have Make installed by typing `make -v` into a terminal. #### Windows + Use the Software Carpentry [Windows installer](https://github.com/swcarpentry/windows-installer). @@ -60,4 +59,7 @@ They can be installed separately, but the easiest approach is to install [Anaconda](https://www.anaconda.com/distribution/) which includes all of the necessary python software. -[zip-file]: {{ page.root }}/files/make-lesson.zip +[zip-file]: files/make-lesson.zip + + + diff --git a/profiles/learner-profiles.md b/profiles/learner-profiles.md new file mode 100644 index 00000000..434e335a --- /dev/null +++ b/profiles/learner-profiles.md @@ -0,0 +1,5 @@ +--- +title: FIXME +--- + +This is a placeholder file. Please add content here. diff --git a/reference.md b/reference.md deleted file mode 100644 index 7b6244f9..00000000 --- a/reference.md +++ /dev/null @@ -1,362 +0,0 @@ ---- -layout: reference ---- - -## Running Make - -To run Make: - -~~~ -$ make -~~~ -{: .language-bash} - -Make will look for a Makefile called `Makefile` and will build the -default target, the first target in the Makefile. - -To use a Makefile with a different name, use the `-f` flag e.g. - -~~~ -$ make -f build-files/analyze.mk -~~~ -{: .language-bash} - -To build a specific target, provide it as an argument e.g. - -~~~ -$ make isles.dat -~~~ -{: .language-bash} - -If the target is up-to-date, Make will print a message like: - -~~~ -make: `isles.dat' is up to date. -~~~ -{: .output} - -To see the actions Make will run when building a target, without -running the actions, use the `--dry-run` flag e.g. - -~~~ -$ make --dry-run isles.dat -~~~ -{: .language-bash} - -Alternatively, use the abbreviation `-n`. - -~~~ -$ make -n isles.dat -~~~ -{: .language-bash} - -## Trouble Shooting - -If Make prints a message like, - -~~~ -Makefile:3: *** missing separator. Stop. -~~~ -{: .error} - -then check that all the actions are indented by TAB characters and not -spaces. - -If Make prints a message like, - -~~~ -No such file or directory: 'books/%.txt' -make: *** [isles.dat] Error 1 -~~~ -{: .error} - -then you may have used the Make wildcard, `%`, in an action in a -pattern rule. Make wildcards cannot be used in actions. - -## Makefiles - -Rules: - -~~~ -target : dependency1 dependency2 ... - action1 - action2 - ... -~~~ -{: .language-make} - -* Each rule has a target, a file to be created, or built. -* Each rule has zero or more dependencies, files that are needed to - build the target. -* `:` separates the target and the dependencies. -* Dependencies are separated by spaces. -* Each rule has zero or more actions, commands to run to build the - target using the dependencies. -* Actions are indented using the TAB character, not 8 spaces. - -Dependencies: - -* If any dependency does not exist then Make will look for a rule to - build it. -* The order of rebuilding dependencies is arbitrary. You should not - assume that they will be built in the order in which they are listed. -* Dependencies must form a directed acyclic graph. A target cannot - depend on a dependency which, in turn depends upon, or has a - dependency which depends upon, that target. - -Comments: - -~~~ -# This is a Make comment. -~~~ -{: .language-make} - -Line continuation character: - -~~~ -ARCHIVE = isles.dat isles.png \ - abyss.dat abyss.png \ - sierra.dat sierra.png -~~~ -{: .language-make} - -* If a list of dependencies or an action is too long, a Makefile can - become more difficult to read. -* Backslash,`\`, the line continuation character, allows you to split - up a list of dependencies or an action over multiple lines, to make - them easier to read. -* Make will combine the multiple lines into a single list of dependencies - or action. - -Phony targets: - -~~~ -.PHONY : clean -clean : - rm -f *.dat -~~~ -{: .language-make} - -* Phony targets are a short-hand for sequences of actions. -* No file with the target name is built when a rule with a phony - target is run. - -Automatic variables: - -* `$<` denotes 'the first dependency of the current rule'. -* `$@` denotes 'the target of the current rule'. -* `$^` denotes 'the dependencies of the current rule'. -* `$*` denotes 'the stem with which the pattern of the current rule matched'. - -Pattern rules: - -~~~ -%.dat : books/%.txt $(COUNT_SRC) - $(COUNT_EXE) $< $@ -~~~ -{: .language-make} - -* The Make wildcard, `%`, specifies a pattern. -* If Make finds a dependency matching the pattern, then the pattern is - substituted into the target. -* The Make wildcard can only be used in targets and dependencies. -* e.g. if Make found a file called `books/abyss.txt`, it would set the - target to be `abyss.dat`. - -Defining and using variables: - -~~~ -COUNT_SRC=wordcount.py -COUNT_EXE=python $(COUNT_SRC) -~~~ -{: .language-make} - -* A variable is assigned a value. For example, `COUNT_SRC` - is assigned the value `wordcount.py`. -* `$(...)` is a reference to a variable. It requests that - Make substitutes the name of a variable for its value. - -Suppress printing of actions: - -~~~ -.PHONY : variables -variables: - @echo TXT_FILES: $(TXT_FILES) -~~~ -{: .language-make} - -* Prefix an action by `@` to instruct Make not to print that action. - -Include the contents of a Makefile in another Makefile: - -~~~ -include config.mk -~~~ -{: .language-make} - -wildcard function: - -~~~ -TXT_FILES=$(wildcard books/*.txt) -~~~ -{: .language-make} - -* Looks for all files matching a pattern e.g. `books/*.txt`, and - return these in a list. -* e.g. `TXT_FILES` is set to `books/abyss.txt books/isles.txt - books/last.txt books/sierra.txt`. - -patsubst ('path substitution') function: - -~~~ -DAT_FILES=$(patsubst books/%.txt, %.dat, $(TXT_FILES)) -~~~ -{: .language-make} - -* Every string that matches `books/%.txt` in `$(TXT_FILES)` is - replaced by `%.dat` and the strings are returned in a list. -* e.g. if `TXT_FILES` is `books/abyss.txt books/isles.txt - books/last.txt books/sierra.txt` this sets `DAT_FILES` to `abyss.dat - isles.dat last.dat sierra.dat`. - -Default targets: - -* In Make version 3.79 the default target is the first target in the - Makefile. -* In Make 3.81, the default target can be explicitly set using the - special variable `.DEFAULT_GOAL` e.g. - -~~~ -.DEFAULT_GOAL := all -~~~ -{: .language-make} - -## Manuals - -[GNU Make Manual][gnu-make-manual]. Reference sections include: - -* [Summary of Options][options-summary] for the `make` command. -* [Quick Reference][quick-reference] of Make directives, text manipulation functions, and special variables. -* [Automatic Variables][automatic-variables]. -* [Special Built-in Target Names][special-targets] - -## Glossary - -{:auto_ids} -action -: The steps a [build manager](#build-manager) must take to create or - update a file or other object. - -assignment -: A request that [Make](#make) stores something in a - [variable](#variable). - -automatic variable -: A variable whose value is automatically redefined for each - [rule](#rule). [Make](#make)'s automatic variables include `$@`, - which holds the rule's [target](#target), `$^`, which holds its - [dependencies](#dependency), and, `$<`, which holds the first of - its dependencies, and `$*`, which holds the [stem](#stem) with which - the pattern was matched. Automatic variables are typically used in - [pattern rules](#pattern-rule). - -build file -: A description of [dependencies](#dependency) and [rules](#rule) - for a [build manager](#build-manager). - -build manager -: A program, such as [Make](#make), whose main purpose is to build or - update software, documentation, web sites, data files, images, and - other things. - -default rule -: The [rule](#rule) that is executed if no [target](#target) is - specified when a [build manager](#build-manager) is run. - -default target -: The [target](#target) of the [default rule](#default-rule). - -dependency -: A file that a [target](#target) depends on. If any of a target's - [dependencies](#dependency) are newer than the target itself, the - target needs to be updated. A target's dependencies are also - called its prerequisites. If a target's dependencies do not exist, - then they need to be built first. - -false dependency -: This can refer to a [dependency](#dependency) that is artificial. - e.g. a false dependency is introduced if a data analysis script - is added as a dependency to the data files that the script - analyses. - -function -: A built-in [Make](#make) utility that performs some operation, for - example gets a list of files matching a pattern. - -incremental build -: The feature of a [build manager](#build-manager) by - which it only rebuilds files that, either directory - or indirectly, depend on a file that was changed. - -macro -: Used as a synonym for [variable](#variable) in certain versions of - [Make](#make). - -Make -: A popular [build manager](#build-manager), from GNU, created in 1977. - -Makefile -: A [build file](#build-file) used by [Make](#make), which, by - default, are named `Makefile`. - -pattern rule -: A [rule](#rule) that specifies a general way to build or update an - entire class of files that can be managed the same way. For - example, a pattern rule can specify how to compile any C file - rather than a single, specific C file, or, to analyze any data - file rather than a single, specific data file. Pattern rules - typically make use of [automatic variables](#automatic-variable) - and [wildcards](#wildcard). - -phony target -: A [target](#target) that does not correspond to a file or other - object. Phony targets are usually symbolic names for sequences of - [actions](#action). - -prerequisite -: A synonym for [dependency](#dependency). - -reference -: A request that [Make](#make) substitutes the name of a - [variable](#variable) for its value. - -rule -: A specification of a [target](#target)'s - [dependencies](#dependency) and what [actions](#action) need to be - executed to build or update the target. - -stem -: The part of the target that was matched by the pattern rule. If - the target is `file.dat` and the target pattern was `%.dat`, then - the stem `$*` is `file`. - -target -: A thing to be created or updated, for example a file. Targets can - have [dependencies](#dependency) that must exist, and be - up-to-date, before the target itself can be built or updated. - -variable -: A symbolic name for something in a [Makefile](#makefile). - -wildcard -: A pattern that can be specified in [dependencies](#dependency) and - [targets](#target). If [Make](#make) finds a dependency matching - the pattern, then the pattern is substituted into the - target. wildcards are often used in [pattern - rules](#pattern-rule). The Make wildcard is `%`. - -[automatic-variables]: https://www.gnu.org/software/make/manual/html_node/Automatic-Variables.html -[gnu-make-manual]: https://www.gnu.org/software/make/manual/ -[options-summary]: https://www.gnu.org/software/make/manual/html_node/Options-Summary.html -[quick-reference]: https://www.gnu.org/software/make/manual/html_node/Quick-Reference.html -[special-targets]: https://www.gnu.org/software/make/manual/html_node/Special-Targets.html diff --git a/site/README.md b/site/README.md new file mode 100644 index 00000000..42997e3d --- /dev/null +++ b/site/README.md @@ -0,0 +1,2 @@ +This directory contains rendered lesson materials. Please do not edit files +here.