Next: , Previous: , Up: Integration with existing software  

Downloading service

Previous sections tell about manual downloading and sending results to remote node. But one wish to remotely initiate downloading. That can be easily solved with exec handles.

exec: {
  warcer: ["/bin/sh", "/path/to/"]
  wgeter: ["/bin/sh", "/path/to/"]
  aria2c: [
    "--on-download-complete", "",
    "--on-bt-download-complete", ""
} contents:

#!/bin/sh -ex

name="$1"-$(date '+%Y%M%d%H%m%S')
read cmdline

tmp=$(mktemp -d)
cd $tmp
wget \
    --page-requisites \
    --convert-links \
    --adjust-extension \
    --restrict-file-names=ascii \
    --span-hosts \
    --random-wait \
    --execute robots=off \
    --reject '*.woff*,*.ttf,*.eot,*.js' \
    --tries 10 \
    --warc-file "$name" \
    --no-warc-compression \
    --no-warc-keep-log \
    $cmdline || :
zstd --rm "$name".warc
nncp-file -nice $NNCP_NICE "$name".warc.zst $NNCP_SENDER:
rm -r $tmp contents:

#!/bin/sh -ex

name=$1-$(date '+%Y%M%d%H%m%S')
read cmdline

wget --output-document=$tmp $cmdline
zstd --rm $tmp
nncp-file -nice $NNCP_NICE $tmp.zst $NNCP_SENDER:$name.zst
rm $tmp.zst

Now you can queue that node to send you some website’s page, file or BitTorrents:

$ echo |
    nncp-exec remote.node warcer postfix-whole-page
$ echo |
    nncp-exec remote.node wgeter postfix-html-page
$ echo \ |
    nncp-exec remote.node aria2c