feat: tool and scripts to interactively explore webgraph

Update scripts to download webgraphs and build the vertex map:
support host-level webgraphs which are shipped with a list of
vertex files.

Author: sebastian-nagel

src/script/webgraph_ranking/graph_explore_build_vertex_map.sh

@@ -11,6 +11,10 @@ if ! shift 2; then
     echo "Build node indexes to interactively explore a Common Crawl webgraph."
     echo "The webgraph files are expected to be placed in the current directory."
     echo
+    echo "  <name>      basename of the graph (without the .graph suffix)"
+    echo "  <vertices>  vertices file name (including the file suffix)"
+    echo "              or directory containing the vertices files"
+    echo
     exit 1
 fi
 
@@ -98,8 +102,13 @@ if [ -e $NAME.iepm ]; then
     index_status
     exit 0
 fi
+CAT_VERTICES="zcat $VERTICES"
+if [ -d $VERTICES ]; then
+    # host-level webgraph, multiple vertex files
+    CAT_VERTICES="zcat $VERTICES/*.txt.gz"
+fi
 if (set -eo pipefail;
-    zcat $VERTICES \
+    eval $CAT_VERTICES \
         | cut -f2 \
         | "$WG" it.unimi.dsi.util.ImmutableExternalPrefixMap -b4Ki $NAME.iepm); then
     echo "immutable external prefix map successfully built: $NAME.iepm"

src/script/webgraph_ranking/graph_explore_download_webgraph.sh

@@ -17,6 +17,17 @@ export LC_ALL=C
 
 BIN="$(dirname $0)"
 
+USING_CURL=false
+USING_WGET=false
+if command -v curl &>/dev/null; then
+    USING_CURL=true
+elif command -v wget &>/dev/null; then
+    USING_WGET=true
+else
+    echo "Either curl or wget are required for downloading" >&2
+    exit 1
+fi
+
 declare -A suffix_name_map
 suffix_name_map=(
     graph      "webgraph / BVGraph"
@@ -33,6 +44,8 @@ function list_webgraph_files() {
         if [ -e $name.$suffix ]; then
             printf " .%-10s : %-20s  (%s)\n" "$suffix" \
                    "${suffix_name_map[$suffix]}" "$name.$suffix"
+        elif [ -d "$name" ] && [[ "$suffix" =~ ^\*. ]]; then
+            ls "$name"/* | sed 's/^/\t/'
         else
             echo -e "Missing $name.$suffix (${suffix_name_map[$suffix]})"
             ok=false
@@ -50,8 +63,20 @@ function download_file() {
     fi
     URL="https://data.commoncrawl.org/projects/hyperlinkgraph/$BASE_NAME/$GRAPH_AGGR_LEVEL/$FILE"
     echo "Downloading $URL"
-    # wget --continue --timestamping "$URL"
-    curl --silent --remote-time -o "$FILE" --time-cond "$FILE" --continue-at - "$URL"
+
+    if $USING_CURL; then
+
+        curl --silent --remote-time -o "$FILE" --time-cond "$FILE" --continue-at - "$URL"
+
+    elif $USING_WGET; then
+
+        if [ "$(dirname "$FILE")" == "." ]; then
+            wget --continue --timestamping "$URL"
+        else
+            wget --continue --timestamping --directory-prefix="$(dirname "$FILE")" "$URL"
+        fi
+
+    fi
 }
 
 function download_files() {
@@ -63,18 +88,33 @@ function download_files() {
 
 
 BASE_NAME="${NAME%-domain}"
+BASE_NAME="${BASE_NAME%-host}"
 GRAPH_AGGR_LEVEL="${NAME##*-}"
 
 
 download_files "$NAME" graph properties stats
-download_files "$NAME-vertices" txt.gz
 download_files "$NAME-t" graph properties
 
+if [ $GRAPH_AGGR_LEVEL == "domain" ]; then
+    download_files "$NAME-vertices" txt.gz
+else
+    download_files "$NAME-vertices" paths.gz
+    zcat "$NAME-vertices".paths.gz \
+        | while read path; do
+        file=${path#projects/hyperlinkgraph/$BASE_NAME/$GRAPH_AGGR_LEVEL/}
+        mkdir -p $(dirname "$file")
+        download_file "$file"
+     done
+fi
+
 echo "Downloaded files"
 echo "- webgraph"
 list_webgraph_files $NAME graph properties stats
 echo "- webgraph (transpose)"
 list_webgraph_files $NAME-t graph properties
 echo "- webgraph vertices"
-list_webgraph_files $NAME-vertices txt.gz
-
+if [ $GRAPH_AGGR_LEVEL == "domain" ]; then
+    list_webgraph_files $NAME-vertices txt.gz
+else
+    list_webgraph_files vertices "*.txt.gz"
+fi