From 89aa26e97b4fd85b7045cea7caac0004d61f4ef1 Mon Sep 17 00:00:00 2001 From: malteos Date: Thu, 27 Aug 2026 10:16:37 +0200 Subject: [PATCH] fix(scripts): use `gzip -dc` instead of `zcat` for portability On macOS (and other BSD systems) /usr/bin/zcat is the compress-era tool: it unconditionally appends `.Z` to its argument and cannot read gzip files at all. Every script here feeds it `.gz` input, so the graph exploration and processing workflows fail on macOS: $ zcat cc-main-2026-jun-jul-aug-host-vertices.paths.gz zcat: can't stat: ...paths.gz (...paths.gz.Z): No such file or directory The failure is easy to miss. In graph_explore_download_webgraph.sh the call is the left-hand side of a pipe into a `while` loop, so the pipeline's exit status is the loop's, `set -e` does not fire, and the script continues with an empty vertex list. It only fails later, at the `list_webgraph_files` check, reporting missing vertex files rather than the decompression error that caused them to be missing. `gzip -dc` is equivalent to GNU `zcat` for both `.gz` and `.Z` input and behaves identically on GNU, BSD and busybox, so this is a no-op on Linux. Replaces all 31 call sites, including the two in host2domaingraph.sh comments so the documented examples stay runnable as written. Claude-Session: https://claude.ai/code/session_01UUzQo1VaEMKBQdt9aactiC --- src/script/host2domaingraph.sh | 8 ++++---- src/script/hostgraph/build_hostgraph.sh | 6 +++--- .../graph_explore_build_vertex_map.sh | 8 ++++---- .../graph_explore_download_webgraph.sh | 2 +- .../webgraph_ranking/process_webgraph.sh | 20 +++++++++---------- .../process_webgraph_degrees.sh | 8 ++++---- 6 files changed, 26 insertions(+), 26 deletions(-) diff --git a/src/script/host2domaingraph.sh b/src/script/host2domaingraph.sh index ea51359..989acaf 100755 --- a/src/script/host2domaingraph.sh +++ b/src/script/host2domaingraph.sh @@ -92,14 +92,14 @@ PARALLEL_SORT_THREADS=2 # The initial solution to ensure that the subdomains of "ac.gov.ascension" are not split # into two blocks, was to add an artificial dot temporarily to the end of each host # name during sorting: -# zcat vertices.txt.gz | sed -e 's/$/./' \ +# gzip -dc vertices.txt.gz | sed -e 's/$/./' \ # | sort $SORTOPTS -t$'\t' -k2,2 | sed -e 's/\.$//' # The domain name "ac.gov.ascension" in the example above becomes temporarily # "ac.gov.ascension." and is now sorted after "ac.gov.ascension-island." # # A sort order that keeps hosts/domains of a common suffix in one block can be # also achieved if dots are replaced by commas: -# zcat vertices.txt.gz | tr . , \ +# gzip -dc vertices.txt.gz | tr . , \ # | sort $SORTOPTS -t$'\t' -k2,2 | tr , . # This approach is utilized by the "Sort-friendly URI Reordering Transform" (SURT), # see . @@ -234,9 +234,9 @@ fi org.commoncrawl.webgraph.HostToDomainGraph \ "${FLAGS[@]}" \ $SIZE \ - <(zcat $_VERTICES) \ + <(gzip -dc $_VERTICES) \ >(gzip >"$OUTPUTDIR"/vertices.txt.gz) \ - <(zcat $_EDGES) \ + <(gzip -dc $_EDGES) \ >(sort $SORTOPTS -t$'\t' -k1,1n -k2,2n -s -u | gzip >"$OUTPUTDIR"/edges.txt.gz) LOG__ "Waiting for data to be written to disk..." diff --git a/src/script/hostgraph/build_hostgraph.sh b/src/script/hostgraph/build_hostgraph.sh index 71f8acf..9d155d5 100755 --- a/src/script/hostgraph/build_hostgraph.sh +++ b/src/script/hostgraph/build_hostgraph.sh @@ -115,13 +115,13 @@ function dump_upload_text() ( if [ $n_vertex_files -eq 1 ]; then mv output/$NAME/hostgraph/tmp_vertices/*.gz output/$NAME/hostgraph/vertices.txt.gz else - zcat output/$NAME/hostgraph/tmp_vertices/*.gz | gzip >output/$NAME/hostgraph/vertices.txt.gz + gzip -dc output/$NAME/hostgraph/tmp_vertices/*.gz | gzip >output/$NAME/hostgraph/vertices.txt.gz fi aws s3 cp --no-progress output/$NAME/hostgraph/vertices.txt.gz $S3_OUTPUT_PREFIX/$UPLOAD_NAME/hostgraph/ hadoop fs -copyToLocal "$HDFS_BASE_DIR"/text/$NAME/edges/*.gz output/$NAME/hostgraph/tmp_edges/ sort_input="" for e in output/$NAME/hostgraph/tmp_edges/*.gz; do - sort_input="$sort_input <(zcat $e)" + sort_input="$sort_input <(gzip -dc $e)" done mkdir -p tmp eval "sort --batch-size 96 --buffer-size 4g --parallel 2 --temporary-directory ./tmp/ --compress-program=gzip -t$'\t' -k1,1n -k2,2n --stable --merge $sort_input | gzip >output/$NAME/hostgraph/edges.txt.gz" @@ -151,7 +151,7 @@ function create_input_splits() { if $INCLUDE_ROBOTSTXT_SITEMAP_LINKS; then aws s3 cp --quiet --no-progress s3://commoncrawl/crawl-data/$CRAWL/robotstxt.paths.gz . fi - zcat ./*.paths.gz | shuf >input.txt + gzip -dc ./*.paths.gz | shuf >input.txt NUM_INPUT_PATHS=$(wc -l ("$WG" it.unimi.dsi.sux4j.mph.GOV4Function $NAME.mph) \ | "$WG" it.unimi.dsi.util.FrontCodedStringList --utf8 --ratio 32 $NAME.fcl @@ -135,7 +135,7 @@ fi # build the `smph` file (string map perfect hash) required to # determine whether a node label is present in the `mph` file if ! [ -e $NAME.smph ]; then - zcat $VERTICES \ + gzip -dc $VERTICES \ | cut -f2 \ | "$WG" it.unimi.dsi.util.ShiftAddXorSignedStringMap $NAME.mph $NAME.smph fi diff --git a/src/script/webgraph_ranking/graph_explore_download_webgraph.sh b/src/script/webgraph_ranking/graph_explore_download_webgraph.sh index abf192b..d5983df 100755 --- a/src/script/webgraph_ranking/graph_explore_download_webgraph.sh +++ b/src/script/webgraph_ranking/graph_explore_download_webgraph.sh @@ -104,7 +104,7 @@ if [ "$GRAPH_AGGR_LEVEL" == "domain" ]; then download_files "$NAME-vertices" txt.gz else download_files "$NAME-vertices" paths.gz - zcat "$NAME-vertices".paths.gz \ + gzip -dc "$NAME-vertices".paths.gz \ | while read path; do file=${path#projects/hyperlinkgraph/$BASE_NAME/$GRAPH_AGGR_LEVEL/} mkdir -p $(dirname "$file") diff --git a/src/script/webgraph_ranking/process_webgraph.sh b/src/script/webgraph_ranking/process_webgraph.sh index d0d10fb..7e48dd4 100755 --- a/src/script/webgraph_ranking/process_webgraph.sh +++ b/src/script/webgraph_ranking/process_webgraph.sh @@ -85,7 +85,7 @@ function join_rank() ( ### unpack scores with LAW, join node names via paste, ### assign ranks on sorted lines by nl $LW it.unimi.dsi.law.io.tool.DataInput2Text --type $_DATA_TYPE $_IN - \ - | paste - <(zcat $_VERT | cut -f2$_EXTRA_FIELDS) \ + | paste - <(gzip -dc $_VERT | cut -f2$_EXTRA_FIELDS) \ | sort --batch-size=$SORT_BATCHES --buffer-size=$SORT_BUFFER_SIZE --compress-program=gzip -t$'\t' -k1,1gr --stable \ | nl -w1 -nln \ | gzip >$_OUT @@ -105,9 +105,9 @@ function join_harmonicc_pagerank() ( fi SORTOPTS="$SORT_PARALLEL_THREADS_OPT --batch-size=$SORT_BATCHES --buffer-size=$SORT_BUFFER_SIZE --compress-program=gzip" (echo -e "$HEADER"; - zcat $_IN_HC | sort $SORTOPTS -t$'\t' -k3,3 --unique --stable \ + gzip -dc $_IN_HC | sort $SORTOPTS -t$'\t' -k3,3 --unique --stable \ | join -a1 -a2 -e'---' -t$'\t' -j3 -o1.1,1.2,2.1,2.2,0$_EXTRA_FIELDS - \ - <(zcat $_IN_PR | sort $SORTOPTS -t$'\t' -k3,3 --unique --stable) \ + <(gzip -dc $_IN_PR | sort $SORTOPTS -t$'\t' -k3,3 --unique --stable) \ | sort $SORTOPTS -t$'\t' -k1,1n -s) \ | gzip >$_OUT ) @@ -139,7 +139,7 @@ function join_ranks_in_memory() ( JAVAOPTS="-Xmx${JAVA_HEAP_GB}g" SORTOPTS="$SORT_PARALLEL_THREADS_OPT --batch-size=$SORT_BATCHES --buffer-size=$SORT_BUFFER_SIZE --compress-program=gzip" (echo -e "$HEADER"; - JAVA_OPTS=$JAVA_OPTS $WG org.commoncrawl.webgraph.JoinSortRanks $OPTS <(zcat $_VERT) $_HC $_PR -) \ + JAVA_OPTS=$JAVA_OPTS $WG org.commoncrawl.webgraph.JoinSortRanks $OPTS <(gzip -dc $_VERT) $_HC $_PR -) \ | sort $SORTOPTS -t$'\t' -k1,1n --stable | gzip >$_OUT ) @@ -155,21 +155,21 @@ function join_degrees() ( # _VERT is a directory with multiple vertices files _VERT="$_VERT/*.gz" fi - zcat $_VERT \ + gzip -dc $_VERT \ | cut -f2- \ | paste $FULLNAME.outdegrees $FULLNAME.indegrees - \ | gzip >$FULLNAME-outdegrees-indegrees.txt.gz # top-N out/indegrees (echo -e "$HEADER"; set +o pipefail; - zcat $FULLNAME-outdegrees-indegrees.txt.gz \ + gzip -dc $FULLNAME-outdegrees-indegrees.txt.gz \ | perl -aF'\t' -lne 'print if $F[0] > 1000' \ | sort -k1,1nr \ | head -10000) \ | gzip >$FULLNAME-outdegrees-indegrees-topout.txt.gz (echo -e "$HEADER"; set +o pipefail; - zcat $FULLNAME-outdegrees-indegrees.txt.gz \ + gzip -dc $FULLNAME-outdegrees-indegrees.txt.gz \ | perl -aF'\t' -lne 'print if $F[1] > 1000' \ | sort -k2,2nr \ | head -10000) \ @@ -205,7 +205,7 @@ if [ -d $EDGES ]; then # edges is a directory with multiple files sort_input="" for e in $EDGES/part-*.gz; do - sort_input="$sort_input <(zcat $e)" + sort_input="$sort_input <(gzip -dc $e)" done if ${USE_WEBGRAPH_BIG:-false}; then ## TODO: @@ -231,10 +231,10 @@ if [ -d $EDGES ]; then else if ${USE_WEBGRAPH_BIG:-false}; then _step bvgraph \ - bash -c "zcat $EDGES | $WG $WGP.BVGraph --once -g $WGP.ArcListASCIIGraph - $FULLNAME" + bash -c "gzip -dc $EDGES | $WG $WGP.BVGraph --once -g $WGP.ArcListASCIIGraph - $FULLNAME" else _step bvgraph \ - $WG $WGP.BVGraph --threads $THREADS -g $WGP.ArcListASCIIGraph <(zcat $EDGES) $FULLNAME + $WG $WGP.BVGraph --threads $THREADS -g $WGP.ArcListASCIIGraph <(gzip -dc $EDGES) $FULLNAME fi fi diff --git a/src/script/webgraph_ranking/process_webgraph_degrees.sh b/src/script/webgraph_ranking/process_webgraph_degrees.sh index bd1dcb7..2793022 100755 --- a/src/script/webgraph_ranking/process_webgraph_degrees.sh +++ b/src/script/webgraph_ranking/process_webgraph_degrees.sh @@ -22,9 +22,9 @@ fi if [ "$TYPE" == "domain" ]; then - zcat $NAME-vertices.txt.gz + gzip -dc $NAME-vertices.txt.gz else - zcat vertices/*.txt.gz + gzip -dc vertices/*.txt.gz fi \ | cut -f2- \ | paste $NAME.outdegrees $NAME.indegrees - \ @@ -38,7 +38,7 @@ fi (echo -e "$HEADER"; set +o pipefail; - zcat $NAME-outdegrees-indegrees.txt.gz \ + gzip -dc $NAME-outdegrees-indegrees.txt.gz \ | perl -aF'\t' -lne 'print if $F[0] > 1000' \ | sort -k1,1nr \ | head -10000) \ @@ -46,7 +46,7 @@ fi (echo -e "$HEADER"; set +o pipefail; - zcat $NAME-outdegrees-indegrees.txt.gz \ + gzip -dc $NAME-outdegrees-indegrees.txt.gz \ | perl -aF'\t' -lne 'print if $F[1] > 1000' \ | sort -k2,2nr \ | head -10000) \