commoncrawl · malteos · Aug 27, 2026
diff --git a/src/script/host2domaingraph.sh b/src/script/host2domaingraph.sh
@@ -92,14 +92,14 @@ PARALLEL_SORT_THREADS=2
 #   The initial solution to ensure that the subdomains of "ac.gov.ascension" are not split
 #   into two blocks, was to add an artificial dot temporarily to the end of each host
 #   name during sorting:
-#     zcat vertices.txt.gz | sed -e 's/$/./' \
+#     gzip -dc vertices.txt.gz | sed -e 's/$/./' \
 #        | sort $SORTOPTS -t$'\t' -k2,2 | sed -e 's/\.$//'
 #   The domain name "ac.gov.ascension" in the example above becomes temporarily
 #   "ac.gov.ascension." and is now sorted after "ac.gov.ascension-island."
 #
 #   A sort order that keeps hosts/domains of a common suffix in one block can be
 #   also achieved if dots are replaced by commas:
-#     zcat vertices.txt.gz | tr . , \
+#     gzip -dc vertices.txt.gz | tr . , \
 #        | sort $SORTOPTS -t$'\t' -k2,2 | tr , .
 #   This approach is utilized by the "Sort-friendly URI Reordering Transform" (SURT),
 #   see <http://crawler.archive.org/articles/user_manual/glossary.html#surt>.
@@ -234,9 +234,9 @@ fi
                     org.commoncrawl.webgraph.HostToDomainGraph \
                     "${FLAGS[@]}" \
                     $SIZE \
-                    <(zcat $_VERTICES) \
+                    <(gzip -dc $_VERTICES) \
                     >(gzip >"$OUTPUTDIR"/vertices.txt.gz) \
-                    <(zcat $_EDGES) \
+                    <(gzip -dc $_EDGES) \
                     >(sort $SORTOPTS -t$'\t' -k1,1n -k2,2n -s -u | gzip >"$OUTPUTDIR"/edges.txt.gz)
 
 LOG__ "Waiting for data to be written to disk..."

diff --git a/src/script/hostgraph/build_hostgraph.sh b/src/script/hostgraph/build_hostgraph.sh
@@ -115,13 +115,13 @@ function dump_upload_text() (
     if [ $n_vertex_files -eq 1 ]; then
         mv output/$NAME/hostgraph/tmp_vertices/*.gz output/$NAME/hostgraph/vertices.txt.gz
     else
-        zcat output/$NAME/hostgraph/tmp_vertices/*.gz | gzip >output/$NAME/hostgraph/vertices.txt.gz
+        gzip -dc output/$NAME/hostgraph/tmp_vertices/*.gz | gzip >output/$NAME/hostgraph/vertices.txt.gz
     fi
     aws s3 cp --no-progress output/$NAME/hostgraph/vertices.txt.gz $S3_OUTPUT_PREFIX/$UPLOAD_NAME/hostgraph/
     hadoop fs -copyToLocal "$HDFS_BASE_DIR"/text/$NAME/edges/*.gz output/$NAME/hostgraph/tmp_edges/
     sort_input=""
     for e in output/$NAME/hostgraph/tmp_edges/*.gz; do
-        sort_input="$sort_input <(zcat $e)"
+        sort_input="$sort_input <(gzip -dc $e)"
     done
     mkdir -p tmp
     eval "sort --batch-size 96 --buffer-size 4g --parallel 2 --temporary-directory ./tmp/ --compress-program=gzip -t$'\t' -k1,1n -k2,2n --stable --merge $sort_input | gzip >output/$NAME/hostgraph/edges.txt.gz"
@@ -151,7 +151,7 @@ function create_input_splits() {
         if $INCLUDE_ROBOTSTXT_SITEMAP_LINKS; then
             aws s3 cp --quiet --no-progress s3://commoncrawl/crawl-data/$CRAWL/robotstxt.paths.gz .
         fi
-        zcat ./*.paths.gz | shuf >input.txt
+        gzip -dc ./*.paths.gz | shuf >input.txt
         NUM_INPUT_PATHS=$(wc -l <input.txt)
         NUM_SPLITS=$((1+NUM_INPUT_PATHS/MAX_INPUT_SIZE))
         if [ $NUM_SPLITS -gt 0 ]; then

diff --git a/src/script/webgraph_ranking/graph_explore_build_vertex_map.sh b/src/script/webgraph_ranking/graph_explore_build_vertex_map.sh
@@ -102,10 +102,10 @@ if [ -e $NAME.iepm ]; then
     index_status
     exit 0
 fi
-CAT_VERTICES="zcat $VERTICES"
+CAT_VERTICES="gzip -dc $VERTICES"
 if [ -d $VERTICES ]; then
     # host-level webgraph, multiple vertex files
-    CAT_VERTICES="zcat $VERTICES/*.txt.gz"
+    CAT_VERTICES="gzip -dc $VERTICES/*.txt.gz"
 fi
 if (set -eo pipefail;
     eval $CAT_VERTICES \
@@ -126,7 +126,7 @@ fi
 #   (reversed domain name) to node ID
 # - a front coded list to map node IDs to node labels
 if ! [ -e $NAME.mph ] || ! [ -e $NAME.fcl ]; then
-    zcat $VERTICES \
+    gzip -dc $VERTICES \
         | cut -f2 \
         | tee >("$WG" it.unimi.dsi.sux4j.mph.GOV4Function $NAME.mph) \
         | "$WG" it.unimi.dsi.util.FrontCodedStringList --utf8 --ratio 32 $NAME.fcl
@@ -135,7 +135,7 @@ fi
 # build the `smph` file (string map perfect hash) required to
 # determine whether a node label is present in the `mph` file
 if ! [ -e $NAME.smph ]; then
-    zcat $VERTICES \
+    gzip -dc $VERTICES \
         | cut -f2 \
         | "$WG" it.unimi.dsi.util.ShiftAddXorSignedStringMap $NAME.mph $NAME.smph
 fi

diff --git a/src/script/webgraph_ranking/graph_explore_download_webgraph.sh b/src/script/webgraph_ranking/graph_explore_download_webgraph.sh
@@ -104,7 +104,7 @@ if [ "$GRAPH_AGGR_LEVEL" == "domain" ]; then
     download_files "$NAME-vertices" txt.gz
 else
     download_files "$NAME-vertices" paths.gz
-    zcat "$NAME-vertices".paths.gz \
+    gzip -dc "$NAME-vertices".paths.gz \
         | while read path; do
         file=${path#projects/hyperlinkgraph/$BASE_NAME/$GRAPH_AGGR_LEVEL/}
         mkdir -p $(dirname "$file")

diff --git a/src/script/webgraph_ranking/process_webgraph.sh b/src/script/webgraph_ranking/process_webgraph.sh
@@ -85,7 +85,7 @@ function join_rank() (
     ### unpack scores with LAW, join node names via paste,
     ### assign ranks on sorted lines by nl
     $LW it.unimi.dsi.law.io.tool.DataInput2Text --type $_DATA_TYPE $_IN - \
-        | paste - <(zcat $_VERT | cut -f2$_EXTRA_FIELDS) \
+        | paste - <(gzip -dc $_VERT | cut -f2$_EXTRA_FIELDS) \
         | sort --batch-size=$SORT_BATCHES --buffer-size=$SORT_BUFFER_SIZE --compress-program=gzip -t$'\t' -k1,1gr --stable \
         | nl -w1 -nln \
         | gzip >$_OUT
@@ -105,9 +105,9 @@ function join_harmonicc_pagerank() (
     fi
     SORTOPTS="$SORT_PARALLEL_THREADS_OPT --batch-size=$SORT_BATCHES --buffer-size=$SORT_BUFFER_SIZE --compress-program=gzip"
     (echo -e "$HEADER";
-     zcat $_IN_HC | sort $SORTOPTS -t$'\t' -k3,3 --unique --stable \
+     gzip -dc $_IN_HC | sort $SORTOPTS -t$'\t' -k3,3 --unique --stable \
          | join -a1 -a2 -e'---' -t$'\t' -j3 -o1.1,1.2,2.1,2.2,0$_EXTRA_FIELDS - \
-                <(zcat $_IN_PR | sort $SORTOPTS -t$'\t' -k3,3 --unique --stable) \
+                <(gzip -dc $_IN_PR | sort $SORTOPTS -t$'\t' -k3,3 --unique --stable) \
          | sort $SORTOPTS -t$'\t' -k1,1n -s) \
      | gzip >$_OUT
 )
@@ -139,7 +139,7 @@ function join_ranks_in_memory() (
     JAVAOPTS="-Xmx${JAVA_HEAP_GB}g"
     SORTOPTS="$SORT_PARALLEL_THREADS_OPT --batch-size=$SORT_BATCHES --buffer-size=$SORT_BUFFER_SIZE --compress-program=gzip"
     (echo -e "$HEADER";
-     JAVA_OPTS=$JAVA_OPTS $WG org.commoncrawl.webgraph.JoinSortRanks $OPTS <(zcat $_VERT) $_HC $_PR -) \
+     JAVA_OPTS=$JAVA_OPTS $WG org.commoncrawl.webgraph.JoinSortRanks $OPTS <(gzip -dc $_VERT) $_HC $_PR -) \
       | sort $SORTOPTS -t$'\t' -k1,1n --stable | gzip >$_OUT
 )
 
@@ -155,21 +155,21 @@ function join_degrees() (
         # _VERT is a directory with multiple vertices files
         _VERT="$_VERT/*.gz"
     fi
-    zcat $_VERT \
+    gzip -dc $_VERT \
         | cut -f2- \
         | paste $FULLNAME.outdegrees $FULLNAME.indegrees - \
         | gzip >$FULLNAME-outdegrees-indegrees.txt.gz
     # top-N out/indegrees
     (echo -e "$HEADER";
      set +o pipefail;
-     zcat $FULLNAME-outdegrees-indegrees.txt.gz \
+     gzip -dc $FULLNAME-outdegrees-indegrees.txt.gz \
          | perl -aF'\t' -lne 'print if $F[0] > 1000' \
          | sort -k1,1nr \
          | head -10000) \
         | gzip >$FULLNAME-outdegrees-indegrees-topout.txt.gz
     (echo -e "$HEADER";
      set +o pipefail;
-     zcat $FULLNAME-outdegrees-indegrees.txt.gz \
+     gzip -dc $FULLNAME-outdegrees-indegrees.txt.gz \
          | perl -aF'\t' -lne 'print if $F[1] > 1000' \
          | sort -k2,2nr \
          | head -10000) \
@@ -205,7 +205,7 @@ if [ -d $EDGES ]; then
     # edges is a directory with multiple files
     sort_input=""
     for e in $EDGES/part-*.gz; do
-        sort_input="$sort_input <(zcat $e)"
+        sort_input="$sort_input <(gzip -dc $e)"
     done
     if ${USE_WEBGRAPH_BIG:-false}; then
         ## TODO:
@@ -231,10 +231,10 @@ if [ -d $EDGES ]; then
 else
     if ${USE_WEBGRAPH_BIG:-false}; then
         _step bvgraph \
-              bash -c "zcat $EDGES | $WG $WGP.BVGraph --once -g $WGP.ArcListASCIIGraph - $FULLNAME"
+              bash -c "gzip -dc $EDGES | $WG $WGP.BVGraph --once -g $WGP.ArcListASCIIGraph - $FULLNAME"
     else
         _step bvgraph \
-              $WG $WGP.BVGraph --threads $THREADS -g $WGP.ArcListASCIIGraph <(zcat $EDGES) $FULLNAME
+              $WG $WGP.BVGraph --threads $THREADS -g $WGP.ArcListASCIIGraph <(gzip -dc $EDGES) $FULLNAME
     fi
 fi
 

diff --git a/src/script/webgraph_ranking/process_webgraph_degrees.sh b/src/script/webgraph_ranking/process_webgraph_degrees.sh
@@ -22,9 +22,9 @@ fi
 
 
 if [ "$TYPE" == "domain" ]; then
-    zcat $NAME-vertices.txt.gz
+    gzip -dc $NAME-vertices.txt.gz
 else
-    zcat vertices/*.txt.gz
+    gzip -dc vertices/*.txt.gz
 fi \
     | cut -f2- \
     | paste $NAME.outdegrees $NAME.indegrees - \
@@ -38,15 +38,15 @@ fi
 
 (echo -e "$HEADER";
  set +o pipefail;
- zcat $NAME-outdegrees-indegrees.txt.gz \
+ gzip -dc $NAME-outdegrees-indegrees.txt.gz \
      | perl -aF'\t' -lne 'print if $F[0] > 1000' \
      | sort -k1,1nr \
      | head -10000) \
     | gzip >$NAME-outdegrees-indegrees-topout.txt.gz
 
 (echo -e "$HEADER";
  set +o pipefail;
- zcat $NAME-outdegrees-indegrees.txt.gz \
+ gzip -dc $NAME-outdegrees-indegrees.txt.gz \
      | perl -aF'\t' -lne 'print if $F[1] > 1000' \
      | sort -k2,2nr \
      | head -10000) \