Downloading service ¶

Previous sections tell about manual downloading and sending results to remote node. But one wish to remotely initiate downloading. That can be easily solved with exec handles.

exec: {
  warcer: ["/bin/sh", "/path/to/warcer.sh"]
  wgeter: ["/bin/sh", "/path/to/wgeter.sh"]
  aria2c: [
    "/usr/local/bin/aria2c",
    "--on-download-complete", "aria2-downloaded.sh",
    "--on-bt-download-complete", "aria2-downloaded.sh"
  ]
}

warcer.sh contents:

#!/bin/sh -ex

name="$1"-$(date '+%Y%M%d%H%m%S')
read cmdline

tmp=$(mktemp -d)
cd $tmp
wget \
    --page-requisites \
    --convert-links \
    --adjust-extension \
    --restrict-file-names=ascii \
    --span-hosts \
    --random-wait \
    --execute robots=off \
    --reject '*.woff*,*.ttf,*.eot,*.js' \
    --tries 10 \
    --warc-file "$name" \
    --no-warc-compression \
    --no-warc-keep-log \
    $cmdline || :
zstd --rm "$name".warc
nncp-file -nice $NNCP_NICE "$name".warc.zst $NNCP_SENDER:
rm -r $tmp

wgeter.sh contents:

#!/bin/sh -ex

name=$1-$(date '+%Y%M%d%H%m%S')
read cmdline

tmp=$(mktemp)
wget --output-document=$tmp $cmdline
zstd --rm $tmp
nncp-file -nice $NNCP_NICE $tmp.zst $NNCP_SENDER:$name.zst
rm $tmp.zst

Now you can queue that node to send you some website’s page, file or BitTorrents:

$ echo http://www.nncpgo.org/Postfix.html |
    nncp-exec remote.node warcer postfix-whole-page
$ echo http://www.nncpgo.org/Postfix.html |
    nncp-exec remote.node wgeter postfix-html-page
$ echo \
    http://www.nncpgo.org/download/nncp-0.11.tar.xz
    http://www.nncpgo.org/download/nncp-0.11.tar.xz.asc |
    nncp-exec remote.node aria2c