From 1be1c1102f1de695bf0e18ce7e8c7a3b80dc01cd Mon Sep 17 00:00:00 2001
From: Gaston Gonzalez <info@thetechprepper.com>
Date: Wed, 25 Dec 2024 17:13:18 -0700
Subject: [PATCH] Offline KB - Added README and a simple download script to
 download a subset of the smaller, topic-specific exports of wikipedia

---
 overlay/etc/skel/wikipedia/README.md | 30 ++++++++++++++++++++
 scripts/download-wikipedia.sh        | 42 ++++++++++++++++++++++++++++
 scripts/install.sh                   |  3 +-
 3 files changed, 74 insertions(+), 1 deletion(-)
 create mode 100644 overlay/etc/skel/wikipedia/README.md
 create mode 100755 scripts/download-wikipedia.sh

diff --git a/overlay/etc/skel/wikipedia/README.md b/overlay/etc/skel/wikipedia/README.md
new file mode 100644
index 0000000..a9500a8
--- /dev/null
+++ b/overlay/etc/skel/wikipedia/README.md
@@ -0,0 +1,30 @@
+# Offline Knowledge Bases
+
+There are several offline repositories that are made freely available
+using a `.zim` file. Depending on the resource, they can be quite large.
+EmComm Tools Community includes an offline reader called "Kiwix" for 
+searching and reading these resources offline.
+
+1. Visit: http://download.kiwix.org/zim/wikipedia/
+
+2. To save space when building your image, consider some of the smaller
+   Wikipedia topic areas, such as:
+
+   * `wikipedia_en_computer_nopic_YYYY-MM.zim`
+   * `wikipedia_en_history_nopic_YYYY-MM.zim`
+   * `wikipedia_en_mathematics_nopic_YYYY-MM.zim`
+   * `wikipedia_en_medicine_nopic_2024-04.zim`
+   * `wikipedia_en_simple_all_nopic_YYYY-MM.zim`
+
+   The full Wikipedia export without images is 56 GB. This file is better
+   stored separately and excluded from the ETC ISO build image.
+
+3. Copy these `.zim` file to `/etc/skel/wikipedia` during the image build.
+
+4. Other high-value offline exports to look into are available at:
+
+   * https://download.kiwix.org/zim/ifixit/
+
+5. Run `kiwix` after a fresh installation. Your .zim files will
+   be available in your home directory in the `wikipedia` folder.
+
diff --git a/scripts/download-wikipedia.sh b/scripts/download-wikipedia.sh
new file mode 100755
index 0000000..1f9baac
--- /dev/null
+++ b/scripts/download-wikipedia.sh
@@ -0,0 +1,42 @@
+#!/bin/bash
+# Author   : Gaston Gonzalez
+# Date     : 25 December 2024
+# Purpose  : Download offline sections of Wikipedia.
+set -e
+
+HTML=kiwix.html
+URL=http://download.kiwix.org/zim/wikipedia
+
+# Location to store .zim files
+ZIM_DIR=/etc/skel/wikipedia
+
+[ ! -e ${ZIM_DIR} ] && mkdir -v ${ZIM_DIR}
+
+if [ ! -e ${HTML} ]; then
+  et-log "Downloading list of English Wikipedia files"
+  curl -s -L -f -o ${HTML} ${URL}
+
+  if [ $? -ne 0 ]; then
+    et-log "Can't download list of files from ${URL}. Exiting..."
+    exit 1
+  fi
+fi
+
+options=()
+while IFS= read -r zim_files; do
+    zim_file=$(echo $zim_files | sed -n 's|.*href="\([^"]*\.zim\)".*|\1|p')
+
+    options+=("$zim_file" "")
+done < <(grep "[.]zim" kiwix.html | grep "wikipedia_en" kiwix.html | grep nopic | grep -E "computer|history|mathematics|medicine|simple" | sort)
+
+#selected_file=$(dialog --clear --menu "Select a Wikipedia file:" 20 80 10 "${options[@]}" 3>&1 1>&2 2>&3)
+selected_file=$(dialog --clear --menu "Select a Wikipedia file:" 20 80 10 "${options[@]}" 3>&1 1>&2 2>&3)
+exit_status=$?
+
+tput sgr 0 && clear
+
+download_url="${URL}/${selected_file}"
+et-log "Downloading ${download_url}..."
+curl -L -f -O ${download_url} && mv ${selected_file} ${ZIM_DIR}
+
+rm ${HTML}
diff --git a/scripts/install.sh b/scripts/install.sh
index bd55584..ce74e1a 100755
--- a/scripts/install.sh
+++ b/scripts/install.sh
@@ -2,7 +2,7 @@
 #
 # Author  : Gaston Gonzalez
 # Date    : 16 March 2024
-# Updated : 23 December 2024
+# Updated : 25 December 2024
 # Purpose : Main installer for EmComm Tools OS Community
 
 . ./env.sh
@@ -55,3 +55,4 @@ exitIfNotRoot
 
 # Add user-specific data
 ./download-osm-maps.sh
+./download-wikipedia.sh