Introduce a "file" element to supersede "datafile"

examples/sample-book/backmatter.xml

@@ -84,11 +84,11 @@ along with PreTeXt.  If not, see <http://www.gnu.org/licenses/>.
     <appendix>
         <title>Some Runestone Elements</title>
         <p>These elements should end up in the Runestone manifest.</p>
-        <datafile filename="datafile-appendix.txt" xml:id="datafile-appendix" label="datafile-appendix">
+        <file filename="datafile-appendix.txt" xml:id="datafile-appendix" label="datafile-appendix" user-interaction="view">
             <pre>
             Ipsum lorem...
             </pre>
-        </datafile>
+        </file>
 
         <program language="py" xml:id="program-appendix" label="program-appendix">
             print("Hello from the appendix!")

examples/sample-book/ext/datafiles/Helper.java

@@ -0,0 +1,24 @@
+/*
+ * This source file and its compiled companion (helper.jar) form a tiny,
+ * self-contained example that exercises PreTeXt's "file" element with
+ * @format = "binary" and @source pointing at an actual compiled Java
+ * archive.  In the sample book (examples/sample-book/rune.xml) a program
+ * with @add-files referencing this file makes the class available to
+ * server-side (Jobe) execution.
+ *
+ * To rebuild the archive after editing this source:
+ *
+ *   javac helper.java
+ *   jar cfe helper.jar Helper helper.class
+ *
+ * PreTeXt stores a base64 (text) representation of the binary file in the
+ * generated "gen/datafile/" directory; the Runestone renderer registers it
+ * with the browser as a hidden payload flagged with "data-isbinary" so a
+ * future Runestone can hand it to a server verbatim.
+ */
+public class Helper {
+    /** The square of n. */
+    public static int square(int n) {
+        return n * n;
+    }
+}

examples/sample-book/gen/datafile/climate-gz.xml

@@ -0,0 +1,2 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<pi:image-b64 xmlns:pi="http://pretextbook.org/2020/pretext/internal" pi:mime-type="application/octet-stream" pi:base64="H4sICGZce2oAA2NsaW1hdGVfc291cmNlLnR4dAAlx70KgCAUhuHdq/jGApE8FeQYEo0tLY0GhxD6Q6vrz2h5eZ+JXZDo12N2K9qHg1sYI29nuusOjMzmEnYgdJuP0R97RNb7xV3pcqGNaSQ0qTpVkaCi+Fwq/blJ1r+NBKlKvLwrP4hvAAAA"/>

examples/sample-book/gen/datafile/helper-jar.xml

@@ -0,0 +1,2 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<pi:image-b64 xmlns:pi="http://pretextbook.org/2020/pretext/internal" pi:mime-type="application/java-archive" pi:base64="UEsDBAoAAAgAAFBfBF0AAAAAAAAAAAAAAAAJAAQATUVUQS1JTkYv/soAAFBLAwQUAAgICABQXwRdAAAAAAAAAAAAAAAAFAAAAE1FVEEtSU5GL01BTklGRVNULk1G803My0xLLS7RDUstKs7Mz7NSMNQz4OVyLkpNLElN0XWqtFIwMtIz0DNS0HDMTazKz9NLzs9V8MxL1tPk5fJNzMzTdc5JLC62UvBIzSlILeLl4uUCAFBLBwheTAJ8VQAAAFMAAABQSwMEFAAICAgAUF8EXQAAAAAAAAAAAAAAAAwAAABIZWxwZXIuY2xhc3M79W/XPgYGBicGfi4GJgZmdgYWHgZWBjZGBoGsxLJE/ZzEvHR9/6Ss1OQSRgY2m8y8zBI7RgZmDc0wdgYOoIhHak5BahEjA4tzfkoqIwO/T2Zeql9pblJqUUhiUg5QhK24sDSxCMhg0fDU9GRk4ArOLy1KTnXLBElyQ7TrgaxiUGRgB7oABJgYGEFuAJKcQJ4skGYE0qxa2xkYNwIZQEOAJBtYUBiohJuBB6pUBqyVgYFFSipjDZpKMSDJCzacDwBQSwcIqBg0nMAAAAD0AAAAUEsBAgoACgAACAAAUF8EXQAAAAAAAAAAAAAAAAkABAAAAAAAAAAAAAAAAAAAAE1FVEEtSU5GL/7KAABQSwECFAAUAAgICABQXwRdXkwCfFUAAABTAAAAFAAAAAAAAAAAAAAAAAArAAAATUVUQS1JTkYvTUFOSUZFU1QuTUZQSwECFAAUAAgICABQXwRdqBg0nMAAAAD0AAAADAAAAAAAAAAAAAAAAADCAAAASGVscGVyLmNsYXNzUEsFBgAAAAADAAMAtwAAALwBAAAAAA=="/>

examples/sample-book/rune.xml

@@ -1185,10 +1185,10 @@ TEST_CASE( "Test the add function" ) {
     <section xml:id="data-files">
         <title>Data Files</title>
 
-        <p>In the following file of climate data, the first column is Year, second column is Global Average Temperature (Celcius), and the third column is Global Emmisions C02 (Giga-tons).  [Normally you might place this inside a block with the <tag>datafile</tag>.]</p>
+        <p>In the following file of climate data, the first column is Year, second column is Global Average Temperature (Celcius), and the third column is Global Emmisions C02 (Giga-tons).  [Normally you might place this inside a block with the <tag>file</tag>.]</p>
 
         <!-- # Section 11.4 Think Like a Computer Scientist, Runestone 2023-01-23 -->
-        <datafile label="file-global-climate-one" xml:id="file-global-climate-one" filename="ccdata1.txt" rows="18" cols="40" editable="yes">
+        <file label="file-global-climate-one" xml:id="file-global-climate-one" filename="ccdata1.txt" rows="18" cols="40" user-interaction="edit">
             <pre>
             1850    -0.37     2.24E-7
             1860    -0.34     3.94E-7
@@ -1209,7 +1209,7 @@ TEST_CASE( "Test the add function" ) {
             2010     0.56     32.7
             2019     0.74     33.3
             </pre>
-        </datafile>
+        </file>
 
         <p>Browser based programs are automatically given access to data files by their filename.</p>
 
@@ -1247,15 +1247,12 @@ TEST_CASE( "Test the add function" ) {
                 <li>LanguageWorkedWith</li>
             </ol></p>
 
-            <datafile label="stack-overflow-survey" filename="so_survey.csv" rows="10" cols="65" editable="no">
-                <pre source="datafiles/stackoverflow.csv"/>
-            </datafile>
+            <file label="stack-overflow-survey" filename="so_survey.csv" rows="10" cols="65" user-interaction="view" format="pre" source="datafiles/stackoverflow.csv"/>
         </data>
 
         <!-- 400px wide, so 400/600 = 66% -->
-        <datafile label="luther-bell" filename="luther-bell.jpg">
-            <image source="datafiles/LutherBellPic.jpg" width="66%"/>
-        </datafile>
+        <file label="luther-bell" filename="luther-bell.jpg" format="image" source="datafiles/LutherBellPic.jpg"/>
+        <image source="datafiles/LutherBellPic.jpg" width="66%"/>
 
         <program label="python-one-pixel" interactive='activecode' language="python">
             <code>
@@ -1270,6 +1267,15 @@ TEST_CASE( "Test the add function" ) {
             </code>
         </program>
 
+        <!-- A text "file" with no @user-interaction stays hidden from -->
+        <!-- the reader (the default); @format may be omitted for a    -->
+        <!-- text file inserted with a "pre" child, taken to be "pre". -->
+        <file label="numbers-hidden" filename="numbers-hidden.txt">
+            <pre>
+10 20 30
+            </pre>
+        </file>
+
         <computation>
             <title>Golden Gate Bridge Image processing</title>
 
@@ -1278,9 +1284,8 @@ TEST_CASE( "Test the add function" ) {
             <p>[Now a data file and a program to process it, all bundled up inside a <tag>computation</tag>, since an <tag>example</tag> gets knowled and the ActiveCode does not fill.]</p>
 
             <!-- 300px wide so 300/600 = 50% -->
-            <datafile label="golden-gate" filename="golden_gate.png">
-                <image source="datafiles/golden-gate-bridge.png" width="50%"/>
-            </datafile>
+            <file label="golden-gate" filename="golden_gate.png" format="image" source="datafiles/golden-gate-bridge.png"/>
+            <image source="datafiles/golden-gate-bridge.png" width="50%"/>
 
             <p>This program changes every pixel of the image.</p>
 
@@ -1314,13 +1319,13 @@ TEST_CASE( "Test the add function" ) {
             </program>
         </computation>
 
-        <p>The examples above all use Python, which will run in your browser.   Other languages will only run when a project is hosted on Runestone Academy servers.  And in this case there is a small twist.  You need to indicate which existing <tag>datafile</tag> your program needs, even if that seems obvious by reading the code.  To do this, make sure each datafile has an <attr>xml:id</attr> and then use the <attr>add-files</attr> attribute on <tag>program</tag> to specify a list of xml:id's that you want to be available.  These datafiles will be written to the program's working directory using the filename specified by the <attr>filename</attr> attribute of the <tag>datafile</tag> element, and are the names you use in your program's code.  As before, no path information is needed, nor allowed.</p>
+        <p>The examples above all use Python, which will run in your browser.   Other languages will only run when a project is hosted on Runestone Academy servers.  And in this case there is a small twist.  You need to indicate which existing <tag>file</tag> your program needs, even if that seems obvious by reading the code.  To do this, make sure each file has an <attr>xml:id</attr> and then use the <attr>add-files</attr> attribute on <tag>program</tag> to specify a list of xml:id's that you want to be available.  These files will be written to the program's working directory using the filename specified by the <attr>filename</attr> attribute of the <tag>file</tag> element, and are the names you use in your program's code.  As before, no path information is needed, nor allowed.</p>
 
         <p>When the <attr>language</attr> attribute of a <tag>program</tag> is set to <c>python3</c> that means in-browser Python is not good enough, and you want the greater power and flexibility of having your code run on a Runestone Academy server.  So this is our first example of using the <attr>datafile</attr> attribute.</p>
 
         <p>The data file is an abbreviated version of the example above, just to be different.  And is not editable.</p>
 
-        <datafile label="file-global-climate-two" xml:id="file-global-climate-two" filename="ccdata2.txt" rows="4" cols="40" editable="no">
+        <file label="file-global-climate-two" xml:id="file-global-climate-two" filename="ccdata2.txt" rows="4" cols="40" user-interaction="view">
             <pre>
             1900    -0.2      2.38
             1910    -0.49     3.34
@@ -1336,7 +1341,7 @@ TEST_CASE( "Test the add function" ) {
             2010     0.56     32.7
             2019     0.74     33.3
             </pre>
-        </datafile>
+        </file>
 
         <p>The program is identical to the above, but we specify <c>python3</c> as the language, and use the smaller file.  So this example is only active when this content is hosted on a Runestone Academy server.</p>
 
@@ -1357,17 +1362,15 @@ TEST_CASE( "Test the add function" ) {
         <data>
             <title>Flowers</title>
 
-            <p>Two flower images as <tag>datafile</tag> for use in upcoming Java program.</p>
+            <p>Two flower images as <tag>file</tag> for use in upcoming Java program.</p>
 
             <!-- Images are 100 x 100.  So width are 100/600 = 17% -->
 
-            <datafile label="flower1-datafile" xml:id="flower1-datafile" filename="flower1.jpg">
-                <image source="datafiles/flower-one.jpeg" width="17%"/>
-            </datafile>
+            <file label="flower1-datafile" xml:id="flower1-datafile" filename="flower1.jpg" format="image" source="datafiles/flower-one.jpeg"/>
+            <image source="datafiles/flower-one.jpeg" width="17%"/>
 
-            <datafile label="flower2-datafile" xml:id="flower2-datafile" filename="flower2.jpg">
-                <image source="datafiles/flower-two.jpeg" width="17%"/>
-            </datafile>
+            <file label="flower2-datafile" xml:id="flower2-datafile" filename="flower2.jpg" format="image" source="datafiles/flower-two.jpeg"/>
+            <image source="datafiles/flower-two.jpeg" width="17%"/>
         </data>
 
         <program label="java-flower-collage" interactive='activecode' language="java" add-files="file-picture-classes-jar, flower1-datafile, flower2-datafile">
@@ -1560,11 +1563,10 @@ TEST_CASE( "Test the add function" ) {
         <!-- https://runestone.academy/ns/books/published/csawesome/Unit8-2DArray/pictureLabA5.html -->
         <!-- https://github.com/bhoffman0/CSAwesome/blob/master/_sources/Unit8-2DArray/pictureLabA5.rst -->
 
-        <datafile label="beach-datafile" xml:id="beach-datafile" filename="beach.jpg">
-            <image source="datafiles/beach.jpg" width="17%"/>
-        </datafile>
+        <file label="beach-datafile" xml:id="beach-datafile" filename="beach.jpg" format="image" source="datafiles/beach.jpg"/>
+        <image source="datafiles/beach.jpg" width="17%"/>
 
-        <datafile label="file-picture-classes-jar" xml:id="file-picture-classes-jar" filename="pictureClasses1.jar" editable="no" hide="yes">
+        <file label="file-picture-classes-jar" xml:id="file-picture-classes-jar" filename="pictureClasses1.jar" user-interaction="none">
             <pre>
             <![CDATA[
             import java.awt.Image;
@@ -2667,7 +2669,7 @@ TEST_CASE( "Test the add function" ) {
             ]]>
             </pre>
             <!-- <pre source="datafiles/picture-classes-library.jar"/> -->
-        </datafile>
+        </file>
 
         <program label="java-only-blue" interactive='activecode' language="java" add-files="file-picture-classes-jar, beach-datafile">
             <code>
@@ -2854,6 +2856,52 @@ TEST_CASE( "Test the add function" ) {
                 ]]>
             </tests>
         </program>
+
+        <!-- A compiled Java archive, stored as an external binary source,  -->
+        <!-- made available to a server-side (Jobe) Java program via        -->
+        <!-- @compile-also.  The source is examples/sample-book/ext/datafiles/ -->
+        <!-- Helper.java; PreTeXt stores a base64 representation in the     -->
+        <!-- generated "gen/datafile/" directory.                           -->
+        <!--                                                               -->
+        <!-- @compile-also both delivers the file (it is folded into the    -->
+        <!-- program's add-files) and marks it as part of the build.  On a   -->
+        <!-- Jobe server this puts the jar on the javac/java classpath      -->
+        <!-- (-cp) so the compiled classes it holds are visible to the      -->
+        <!-- program.  On other languages the meaning of "add to            -->
+        <!-- compilation" differs: text source files are compiled           -->
+        <!-- together, C/C++ object/archive files are placed on the link    -->
+        <!-- line, and interpreted languages just read the delivered file   -->
+        <!-- from the working directory.                                    -->
+        <file label="helper-jar" xml:id="helper-jar" filename="helper.jar" format="binary" source="datafiles/helper.jar"/>
+
+        <program label="java-helper-square" interactive="activecode" language="java" compile-also="helper-jar">
+            <code>
+            public class Main {
+                public static void main(String[] args) {
+                    System.out.println(Helper.square(5));
+                    System.out.println(Helper.square(-4));
+                }
+            }
+            </code>
+        </program>
+
+        <!-- A binary data file (gzip-compressed text) delivered to a        -->
+        <!-- server-side (Jobe) program with plain @add-files, showing that  -->
+        <!-- a binary "file" is also just data in the working directory.     -->
+        <!-- The program decompresses it, proving the base64 round-trip      -->
+        <!-- preserved the bytes exactly.  No @compile-also here: Python has  -->
+        <!-- no build step to add the file to.                               -->
+        <file label="climate-gz" xml:id="climate-gz" filename="climate.txt.gz" format="binary" source="datafiles/climate.gz"/>
+
+        <program label="python-read-gzip" interactive="activecode" language="python3" add-files="climate-gz">
+            <code>
+            import gzip
+
+            with gzip.open("climate.txt.gz", "rt") as f:
+                for line in f:
+                    print(line, end="")
+            </code>
+        </program>
     </section>
 
     <section xml:id="videos-in-runestone">
@@ -3481,9 +3529,8 @@ TEST_CASE( "Test the add function" ) {
                 <p>After you check a correct answer you will be able to Run the code you created - it will be used to modify the image shown below.</p>
 
                 <!-- 300px wide so 300/600 = 50% -->
-                <datafile label="golden-gate2" filename="golden-gate.png">
-                    <image source="datafiles/golden-gate-bridge.png" width="50%"/>
-                </datafile>
+                <file label="golden-gate2" filename="golden-gate.png" format="image" source="datafiles/golden-gate-bridge.png"/>
+                <image source="datafiles/golden-gate-bridge.png" width="50%"/>
 
             </statement>
             <blocks>

pretext/lib/pretext.py

@@ -898,6 +898,23 @@ def datafiles_to_xml(xml_source, pub_file, stringparams, xmlid_root, dest_dir):
                 log.info("PTX:WARNING : MIME type of image {} not determined".format(data_file))
                 mime_type = "unknown"
 
+            # Open binary file and encode in base64 with standard module
+            with open(data_file, "rb") as f:
+                base64version = base64.b64encode(f.read()).decode("utf8")
+            xml_representation = image_info.format(mime_type, base64version)
+        elif file_type == "binary":
+            # best guess of the MIME type, based on the filename extension
+            _, extension = os.path.splitext(data_file)
+            # normalize, drop leading period
+            # in rough popularity order
+            lcext = extension[1:].lower()
+            if lcext == "zip":
+                mime_type = "application/zip"
+            elif lcext == "jar":
+                mime_type = "application/java-archive"
+            else:
+                mime_type = "application/octet-stream"
+
             # Open binary file and encode in base64 with standard module
             with open(data_file, "rb") as f:
                 base64version = base64.b64encode(f.read()).decode("utf8")

schema/pretext.xml

@@ -2917,6 +2917,27 @@
                 }
             BlockStatement |= Datafile
 
+# File element used to provide support files for programs.
+             # A "pre" child names a text file, so @format is optional
+             # (and necessarily "pre") when it is present, defaulting
+             # accordingly.  An external @source file is otherwise
+             # required to carry an explicit @format.
+             File =
+                 element file {
+                     UniqueID?,
+                     LabelID?,
+                     Component?,
+                     attribute filename {text},
+                     attribute rows {xsd:integer}?,
+                     attribute cols {xsd:integer}?,
+                     attribute user-interaction {"none" | "view" | "edit"}?,
+                     (
+                         (attribute format {"pre"}?, element pre {text}) |
+                         (attribute format {"binary" | "image" | "pre"}, attribute source {text})
+                     )
+                 }
+             BlockStatement |= File
+
             # Query (poll/survey) element; results are private to the
             # instructor unless @visibility says otherwise
             Query =

xsl/extract-datafile.xsl

@@ -68,4 +68,27 @@ along with PreTeXt.  If not, see <http://www.gnu.org/licenses/>.
     <xsl:text>&#xa;</xsl:text>
 </xsl:template>
 
+<!-- The "file" element replaces "datafile".  It names its kind with -->
+<!-- @format (binary, image, or pre) instead of a child element, and -->
+<!-- gives a path to an external file with @source.  A "file" with   -->
+<!-- only a "pre" child has no external file, so it needs no line    -->
+<!-- here (its content is used directly, like "datafile/pre" without -->
+<!-- @source).                                                       -->
+<xsl:template match="file[@source]" mode="extraction">
+    <!-- 1. identifier -->
+    <xsl:apply-templates select="." mode="assembly-id" />
+    <xsl:text> </xsl:text>
+    <!-- 2. Type from @format, defaulting to "pre" -->
+    <xsl:choose>
+        <xsl:when test="@format">
+            <xsl:value-of select="@format"/>
+        </xsl:when>
+        <xsl:otherwise>pre</xsl:otherwise>
+    </xsl:choose>
+    <xsl:text> </xsl:text>
+    <!-- 3. path relative to external directory -->
+    <xsl:value-of select="@source"/>
+    <xsl:text>&#xa;</xsl:text>
+</xsl:template>
+
 </xsl:stylesheet>