From fbe2934718f99806008af5a07d32c81c1405578e Mon Sep 17 00:00:00 2001 From: NunoSempere Date: Mon, 28 Jun 2021 12:27:08 +0200 Subject: [PATCH] First commit --- debian/changelog | 5 +++ debian/control | 15 +++++++ debian/copyright | 40 +++++++++++++++++++ debian/install | 1 + debian/rules | 25 ++++++++++++ debian/source/format | 1 + longnowformd.sh | 93 ++++++++++++++++++++++++++++++++++++++++++++ 7 files changed, 180 insertions(+) create mode 100644 debian/changelog create mode 100644 debian/control create mode 100644 debian/copyright create mode 100644 debian/install create mode 100755 debian/rules create mode 100644 debian/source/format create mode 100644 longnowformd.sh diff --git a/debian/changelog b/debian/changelog new file mode 100644 index 0000000..c8eb648 --- /dev/null +++ b/debian/changelog @@ -0,0 +1,5 @@ +longnowformd (0.1-1) bionic; urgency=medium + + * Initial release (Closes: #nnnn) + + -- Nuno Sempere Mon, 28 Jun 2021 12:18:50 +0200 diff --git a/debian/control b/debian/control new file mode 100644 index 0000000..a2cc391 --- /dev/null +++ b/debian/control @@ -0,0 +1,15 @@ +Source: longnowformd +Section: utils +Priority: optional +Maintainer: Nuno Sempere +Build-Depends: debhelper-compat (= 12) +Standards-Version: 4.4.1 +Homepage: +#Vcs-Browser: https://salsa.debian.org/debian/longnowformd +#Vcs-Git: https://salsa.debian.org/debian/longnowformd.git + +Package: longnowformd +Architecture: all +Depends: ${misc:Depends} +Description: + diff --git a/debian/copyright b/debian/copyright new file mode 100644 index 0000000..f672849 --- /dev/null +++ b/debian/copyright @@ -0,0 +1,40 @@ +Format: https://www.debian.org/doc/packaging-manuals/copyright-format/1.0/ +Upstream-Name: longnowformd +Upstream-Contact: 2021 Nuno Sempere +Source: + +Files: * +Copyright: 2021 Nuno Sempere +License: MIT + +Files: debian/* +Copyright: 2021 Nuno Sempere +License: MIT + +License: MIT + Permission is hereby granted, free of charge, to any person obtaining a + copy of this software and associated documentation files (the "Software"), + to deal in the Software without restriction, including without limitation + the rights to use, copy, modify, merge, publish, distribute, sublicense, + and/or sell copies of the Software, and to permit persons to whom the + Software is furnished to do so, subject to the following conditions: + . + The above copyright notice and this permission notice shall be included + in all copies or substantial portions of the Software. + . + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS + OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. + IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY + CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, + TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +# Please also look if there are files or directories which have a +# different copyright/license attached and list them here. +# Please avoid picking licenses with terms that are more restrictive than the +# packaged work, as it may make Debian's contributions unacceptable upstream. +# +# If you need, there are some extra license texts available in two places: +# /usr/share/debhelper/dh_make/licenses/ +# /usr/share/common-licenses/ diff --git a/debian/install b/debian/install new file mode 100644 index 0000000..26393e2 --- /dev/null +++ b/debian/install @@ -0,0 +1 @@ +longnowformd.sh usr/bin diff --git a/debian/rules b/debian/rules new file mode 100755 index 0000000..e1c367c --- /dev/null +++ b/debian/rules @@ -0,0 +1,25 @@ +#!/usr/bin/make -f +# See debhelper(7) (uncomment to enable) +# output every command that modifies files on the build system. +#export DH_VERBOSE = 1 + + +# see FEATURE AREAS in dpkg-buildflags(1) +#export DEB_BUILD_MAINT_OPTIONS = hardening=+all + +# see ENVIRONMENT in dpkg-buildflags(1) +# package maintainers to append CFLAGS +#export DEB_CFLAGS_MAINT_APPEND = -Wall -pedantic +# package maintainers to append LDFLAGS +#export DEB_LDFLAGS_MAINT_APPEND = -Wl,--as-needed + + +%: + dh $@ + + +# dh_make generated override targets +# This is example for Cmake (See https://bugs.debian.org/641051 ) +#override_dh_auto_configure: +# dh_auto_configure -- # -DCMAKE_LIBRARY_PATH=$(DEB_HOST_MULTIARCH) + diff --git a/debian/source/format b/debian/source/format new file mode 100644 index 0000000..163aaf8 --- /dev/null +++ b/debian/source/format @@ -0,0 +1 @@ +3.0 (quilt) diff --git a/longnowformd.sh b/longnowformd.sh new file mode 100644 index 0000000..a7eafb4 --- /dev/null +++ b/longnowformd.sh @@ -0,0 +1,93 @@ +function getMdLinks(){ # Use: Takes a markdown file file.md, extracts all links, finds the unique ones and saves them to file.md.links + echo "" + echo "Extracting links..." + + grep -Eoi '\]\((.*)\)' $1 | grep -Eo '(http|https)://[^)]+' >> "$1.links" + ## sed -i 's/www.wikiwand.com\/en/en.wikipedia.org\/wiki/g' $1 + awk '!seen[$0]++' "$1.links" > "$1.links2" && mv "$1.links2" "$1.links" + + echo "Done." + echo "" +} + +function pushToArchive(){ +# Use: Takes a txt file with one link on each line and pushes all the links to the internet archive. Saves those links to a textfile +# References: +# https://unix.stackexchange.com/questions/181254/how-to-use-grep-and-cut-in-script-to-obtain-website-urls-from-an-html-file +# https://github.com/oduwsdl/archivenow +# For the double underscore, see: https://stackoverflow.com/questions/13797087/bash-why-double-underline-for-private-functions-why-for-bash-complet/15181999 + echo "Pushing to archive.org..." + + input=$1 + counter=1 + + rm -f "$1.archived" + touch "$1.archived" + + while IFS= read -r line + do + wait + if [ $(($counter % 15)) -eq 0 ] + then + printf "\nArchive.org doesn't accept more than 15 links per min; sleeping for 1min...\n" + sleep 1m + fi + echo "Url: $line" + archiveURL=$(archivenow --ia $line) + echo $archiveURL >> "$1.archived" + echo $archiveURL + counter=$((counter+1)) + echo "" + done < "$input" + + echo "Done." + echo "" +} + +function addArchiveLinksToFile(){ + + originalFile="$1" + originalFileTemp="$originalFile.temp" + linksFile="$1.links" + archivedLinksFile="$1.links.archived" + longNowFile="$1.longnow" + + echo "Creating longnow file @ $longNowFile..." + + rm -f "$longNowFile" + touch "$longNowFile" + cp "$originalFile" "$originalFileTemp" + + while IFS= read -r url + do + wait + + archivedUrl=$(grep "$url" "$archivedLinksFile" | tail -1) + ## echo "Url: $url" + ## echo "ArchivedUrl: $archivedUrl" + urlForSed="${url//\//\\/}" + archiveUrlForSed="${archivedUrl//\//\\/}" + sed -i "s/$urlForSed)/$urlForSed) ([a]($archiveUrlForSed))/g" "$1" + done < "$linksFile" + mv "$originalFile" "$longNowFile" + mv "$originalFileTemp" "$originalFile" + + echo "Done." + +} + +function longnow(){ + doesArchiveNowExist=$(whereis "archivenow") + if [ "$doesArchiveNowExist" == "archivenow:" ] + then + echo "Required archivenow utility not found in path." + echo "Install with \$ pip install archivenow" + echo "(resp. \$ pip3 install archivenow)" + echo "Or follow instructions on https://github.com/oduwsdl/archivenow" + else + getMdLinks $1 + pushToArchive $1.links + addArchiveLinksToFile $1 + fi +} +