Automatically detect EOL from paste events and output setting

src/web/utils/editorUtils.mjs

@@ -95,3 +95,42 @@ export function escapeControlChars(str, preserveWs=false, lineBreak="\n") {
         return n.outerHTML;
     });
 }
+
+/**
+ * Convert and EOL sequence to its name
+ */
+export const eolSeqToCode = {
+    "\u000a": "LF",
+    "\u000b": "VT",
+    "\u000c": "FF",
+    "\u000d": "CR",
+    "\u000d\u000a": "CRLF",
+    "\u0085": "NEL",
+    "\u2028": "LS",
+    "\u2029": "PS"
+};
+
+/**
+ * Convert an EOL name to its sequence
+ */
+export const eolCodeToSeq = {
+    "LF": "\u000a",
+    "VT": "\u000b",
+    "FF": "\u000c",
+    "CR": "\u000d",
+    "CRLF": "\u000d\u000a",
+    "NEL": "\u0085",
+    "LS": "\u2028",
+    "PS": "\u2029"
+};
+
+export const eolCodeToName = {
+    "LF": "Line Feed",
+    "VT": "Vertical Tab",
+    "FF": "Form Feed",
+    "CR": "Carriage Return",
+    "CRLF": "Carriage Return + Line Feed",
+    "NEL": "Next Line",
+    "LS": "Line Separator",
+    "PS": "Paragraph Separator"
+};

src/web/utils/statusBar.mjs

@@ -6,6 +6,7 @@
 
 import {showPanel} from "@codemirror/view";
 import {CHR_ENC_SIMPLE_LOOKUP, CHR_ENC_SIMPLE_REVERSE_LOOKUP} from "../../core/lib/ChrEnc.mjs";
+import { eolCodeToName, eolSeqToCode } from "./editorUtils.mjs";
 
 /**
  * A Status bar extension for CodeMirror
@@ -92,22 +93,12 @@ class StatusBarPanel {
         // preventDefault is required to stop the URL being modified and popState being triggered
         e.preventDefault();
 
-        const eolLookup = {
+        const eolCode = e.target.getAttribute("data-val");
-            "LF": "\u000a",
+        if (!eolCode) return;
-            "VT": "\u000b",
-            "FF": "\u000c",
-            "CR": "\u000d",
-            "CRLF": "\u000d\u000a",
-            "NEL": "\u0085",
-            "LS": "\u2028",
-            "PS": "\u2029"
-        };
-        const eolval = eolLookup[e.target.getAttribute("data-val")];
-
-        if (eolval === undefined) return;
 
         // Call relevant EOL change handler
-        this.eolHandler(eolval);
+        this.eolHandler(e.target.getAttribute("data-val"), true);
+
         hideElement(e.target.closest(".cm-status-bar-select-content"));
     }
 
@@ -223,23 +214,13 @@ class StatusBarPanel {
     updateEOL(state) {
         if (state.lineBreak === this.eolVal) return;
 
-        const eolLookup = {
-            "\u000a": ["LF", "Line Feed"],
-            "\u000b": ["VT", "Vertical Tab"],
-            "\u000c": ["FF", "Form Feed"],
-            "\u000d": ["CR", "Carriage Return"],
-            "\u000d\u000a": ["CRLF", "Carriage Return + Line Feed"],
-            "\u0085": ["NEL", "Next Line"],
-            "\u2028": ["LS", "Line Separator"],
-            "\u2029": ["PS", "Paragraph Separator"]
-        };
-
         const val = this.dom.querySelector(".eol-value");
         const button = val.closest(".cm-status-bar-select-btn");
-        const eolName = eolLookup[state.lineBreak];
+        const eolCode = eolSeqToCode[state.lineBreak];
-        val.textContent = eolName[0];
+        const eolName = eolCodeToName[eolCode];
-        button.setAttribute("title", `End of line sequence:<br>${eolName[1]}`);
+        val.textContent = eolCode;
-        button.setAttribute("data-original-title", `End of line sequence:<br>${eolName[1]}`);
+        button.setAttribute("title", `End of line sequence:<br>${eolName}`);
+        button.setAttribute("data-original-title", `End of line sequence:<br>${eolName}`);
         this.eolVal = state.lineBreak;
     }
 

src/web/waiters/ControlsWaiter.mjs

@@ -5,6 +5,7 @@
  */
 
 import Utils from "../../core/Utils.mjs";
+import { eolSeqToCode } from "../utils/editorUtils.mjs";
 
 
 /**
@@ -140,16 +141,16 @@ class ControlsWaiter {
 
         const inputChrEnc = this.manager.input.getChrEnc();
         const outputChrEnc = this.manager.output.getChrEnc();
-        const inputEOLSeq = this.manager.input.getEOLSeq();
+        const inputEOL = eolSeqToCode[this.manager.input.getEOLSeq()];
-        const outputEOLSeq = this.manager.output.getEOLSeq();
+        const outputEOL = eolSeqToCode[this.manager.output.getEOLSeq()];
 
         const params = [
             includeRecipe ? ["recipe", recipeStr] : undefined,
             includeInput && input.length ? ["input", Utils.escapeHtml(input)] : undefined,
             inputChrEnc !== 0 ? ["ienc", inputChrEnc] : undefined,
             outputChrEnc !== 0 ? ["oenc", outputChrEnc] : undefined,
-            inputEOLSeq !== "\n" ? ["ieol", inputEOLSeq] : undefined,
+            inputEOL !== "LF" ? ["ieol", inputEOL] : undefined,
-            outputEOLSeq !== "\n" ? ["oeol", outputEOLSeq] : undefined
+            outputEOL !== "LF" ? ["oeol", outputEOL] : undefined
         ];
 
         const hash = params

src/web/waiters/InputWaiter.mjs

@@ -42,7 +42,7 @@ import {
 
 import {statusBar} from "../utils/statusBar.mjs";
 import {fileDetailsPanel} from "../utils/fileDetails.mjs";
-import {renderSpecialChar} from "../utils/editorUtils.mjs";
+import {eolCodeToSeq, eolCodeToName, renderSpecialChar} from "../utils/editorUtils.mjs";
 
 
 /**
@@ -62,6 +62,7 @@ class InputWaiter {
 
         this.inputTextEl = document.getElementById("input-text");
         this.inputChrEnc = 0;
+        this.eolSetManually = false;
         this.initEditor();
 
         this.inputWorker = null;
@@ -92,6 +93,7 @@ class InputWaiter {
             fileDetailsPanel: new Compartment
         };
 
+        const self = this;
         const initialState = EditorState.create({
             doc: null,
             extensions: [
@@ -141,6 +143,15 @@ class InputWaiter {
                     if (e.docChanged && !this.silentInputChange)
                         this.inputChange(e);
                     this.silentInputChange = false;
+                }),
+
+                // Event handlers
+                EditorView.domEventHandlers({
+                    paste(event, view) {
+                        setTimeout(() => {
+                            self.afterPaste(event);
+                        });
+                    }
                 })
             ]
         });
@@ -154,12 +165,35 @@ class InputWaiter {
     /**
      * Handler for EOL change events
      * Sets the line separator
-     * @param {string} eolVal
+     * @param {string} eol
+     * @param {boolean} manual - a flag for whether this was set by the user or automatically
      */
-    eolChange(eolVal) {
+    eolChange(eol, manual=false) {
-        const oldInputVal = this.getInput();
+        const eolVal = eolCodeToSeq[eol];
+        if (eolVal === undefined) return;
+
+        const eolBtn = document.querySelector("#input-text .eol-value");
+        if (manual) {
+            this.eolSetManually = true;
+            eolBtn.classList.remove("font-italic");
+        } else {
+            eolBtn.classList.add("font-italic");
+        }
+
+        if (eolVal === this.getEOLSeq()) return;
+
+        if (!manual) {
+            // Pulse
+            eolBtn.classList.add("pulse");
+            setTimeout(() => {
+                eolBtn.classList.remove("pulse");
+            }, 2000);
+            // Alert
+            this.app.alert(`Input EOL separator has been changed to ${eolCodeToName[eol]}`, 5000);
+        }
 
         // Update the EOL value
+        const oldInputVal = this.getInput();
         this.inputEditorView.dispatch({
             effects: this.inputEditorConf.eol.reconfigure(EditorState.lineSeparator.of(eolVal))
         });
@@ -866,6 +900,49 @@ class InputWaiter {
         }, delay, "inputChange", this, [e])();
     }
 
+    /**
+     * Handler that fires just after input paste events.
+     * Checks whether the EOL separator or character encoding should be updated.
+     *
+     * @param {event} e
+     */
+    afterPaste(e) {
+        // If EOL has been fixed, skip this.
+        if (this.eolSetManually) return;
+
+        const inputText = this.getInput();
+
+        // Detect most likely EOL sequence
+        const eolCharCounts = {
+            "LF": inputText.count("\u000a"),
+            "VT": inputText.count("\u000b"),
+            "FF": inputText.count("\u000c"),
+            "CR": inputText.count("\u000d"),
+            "CRLF": inputText.count("\u000d\u000a"),
+            "NEL": inputText.count("\u0085"),
+            "LS": inputText.count("\u2028"),
+            "PS": inputText.count("\u2029")
+        };
+
+        // If all zero, leave alone
+        const total = Object.values(eolCharCounts).reduce((acc, curr) => {
+            return acc + curr;
+        }, 0);
+        if (total === 0) return;
+
+        // If CRLF not zero and more than half the highest alternative, choose CRLF
+        const highest = Object.entries(eolCharCounts).reduce((acc, curr) => {
+            return curr[1] > acc[1] ? curr : acc;
+        }, ["LF", 0]);
+        if ((eolCharCounts.CRLF * 2) > highest[1]) {
+            this.eolChange("CRLF");
+            return;
+        }
+
+        // Else choose max
+        this.eolChange(highest[0]);
+    }
+
     /**
      * Handler for input dragover events.
      * Gives the user a visual cue to show that items can be dropped here.
@@ -1199,6 +1276,9 @@ class InputWaiter {
         this.manager.output.removeAllOutputs();
         this.manager.output.terminateZipWorker();
 
+        this.eolSetManually = false;
+        this.manager.output.eolSetManually = false;
+
         const tabsList = document.getElementById("input-tabs");
         const tabsListChildren = tabsList.children;
 

src/web/waiters/OutputWaiter.mjs

@@ -38,7 +38,7 @@ import {
 import {statusBar} from "../utils/statusBar.mjs";
 import {htmlPlugin} from "../utils/htmlWidget.mjs";
 import {copyOverride} from "../utils/copyOverride.mjs";
-import {renderSpecialChar} from "../utils/editorUtils.mjs";
+import {eolCodeToSeq, eolCodeToName, renderSpecialChar} from "../utils/editorUtils.mjs";
 
 
 /**
@@ -70,6 +70,7 @@ class OutputWaiter {
         this.zipWorker = null;
         this.maxTabs = this.manager.tabs.calcMaxTabs();
         this.tabTimeout = null;
+        this.eolSetManually = false;
     }
 
     /**
@@ -146,9 +147,33 @@ class OutputWaiter {
     /**
      * Handler for EOL change events
      * Sets the line separator
-     * @param {string} eolVal
+     * @param {string} eol
+     * @param {boolean} manual - a flag for whether this was set by the user or automatically
      */
-    async eolChange(eolVal) {
+    async eolChange(eol, manual=false) {
+        const eolVal = eolCodeToSeq[eol];
+        if (eolVal === undefined) return;
+
+        const eolBtn = document.querySelector("#output-text .eol-value");
+        if (manual) {
+            this.eolSetManually = true;
+            eolBtn.classList.remove("font-italic");
+        } else {
+            eolBtn.classList.add("font-italic");
+        }
+
+        if (eolVal === this.getEOLSeq()) return;
+
+        if (!manual) {
+            // Pulse
+            eolBtn.classList.add("pulse");
+            setTimeout(() => {
+                eolBtn.classList.remove("pulse");
+            }, 2000);
+            // Alert
+            this.app.alert(`Output EOL separator has been changed to ${eolCodeToName[eol]}`, 5000);
+        }
+
         const currentTabNum = this.manager.tabs.getActiveTab("output");
         if (currentTabNum >= 0) {
             this.outputs[currentTabNum].eolSequence = eolVal;
@@ -276,6 +301,9 @@ class OutputWaiter {
         // If turning word wrap off, do it before we populate the editor for performance reasons
         if (!wrap) this.setWordWrap(wrap);
 
+        // Detect suitable EOL sequence
+        this.detectEOLSequence(data);
+
         // We use setTimeout here to delay the editor dispatch until the next event cycle,
         // ensuring all async actions have completed before attempting to set the contents
         // of the editor. This is mainly with the above call to setWordWrap() in mind.
@@ -345,6 +373,48 @@ class OutputWaiter {
         });
     }
 
+    /**
+     * Checks whether the EOL separator should be updated
+     *
+     * @param {string} data
+     */
+    detectEOLSequence(data) {
+        // If EOL has been fixed, skip this.
+        if (this.eolSetManually) return;
+        // If data is too long, skip this.
+        if (data.length > 1000000) return;
+
+        // Detect most likely EOL sequence
+        const eolCharCounts = {
+            "LF": data.count("\u000a"),
+            "VT": data.count("\u000b"),
+            "FF": data.count("\u000c"),
+            "CR": data.count("\u000d"),
+            "CRLF": data.count("\u000d\u000a"),
+            "NEL": data.count("\u0085"),
+            "LS": data.count("\u2028"),
+            "PS": data.count("\u2029")
+        };
+
+        // If all zero, leave alone
+        const total = Object.values(eolCharCounts).reduce((acc, curr) => {
+            return acc + curr;
+        }, 0);
+        if (total === 0) return;
+
+        // If CRLF not zero and more than half the highest alternative, choose CRLF
+        const highest = Object.entries(eolCharCounts).reduce((acc, curr) => {
+            return curr[1] > acc[1] ? curr : acc;
+        }, ["LF", 0]);
+        if ((eolCharCounts.CRLF * 2) > highest[1]) {
+            this.eolChange("CRLF");
+            return;
+        }
+
+        // Else choose max
+        this.eolChange(highest[0]);
+    }
+
     /**
      * Calculates the maximum number of tabs to display
      */

tests/browser/00_nightwatch.js

@@ -230,6 +230,7 @@ module.exports = {
 
         // Alert bar shows and contains correct content
         browser
+            .waitForElementNotVisible("#snackbar-container")
             .click("#copy-output")
             .waitForElementVisible("#snackbar-container")
             .waitForElementVisible("#snackbar-container .snackbar-content")

tests/browser/01_io.js

@@ -545,8 +545,8 @@ module.exports = {
         browser.expect.element("#output-text .cm-status-bar .stats-lines-value").text.to.equal("2");
 
         /* Line endings appear in the URL */
-        browser.assert.urlContains("ieol=%0D%0A");
+        browser.assert.urlContains("ieol=CRLF");
-        browser.assert.urlContains("oeol=%0D");
+        browser.assert.urlContains("oeol=CR");
 
         /* Preserved when changing tabs */
         browser
@@ -643,7 +643,7 @@ module.exports = {
     "Loading from URL": browser => {
         /* Complex deep link populates the input correctly (encoding, eol, input) */
         browser
-            .urlHash("recipe=To_Base64('A-Za-z0-9%2B/%3D')&input=VGhlIHNoaXBzIGh1bmcgaW4gdGhlIHNreSBpbiBtdWNoIHRoZSBzYW1lIHdheSB0aGF0IGJyaWNrcyBkb24ndC4M&ienc=21866&oenc=1201&ieol=%0C&oeol=%E2%80%A9")
+            .urlHash("recipe=To_Base64('A-Za-z0-9%2B/%3D')&input=VGhlIHNoaXBzIGh1bmcgaW4gdGhlIHNreSBpbiBtdWNoIHRoZSBzYW1lIHdheSB0aGF0IGJyaWNrcyBkb24ndC4M&ienc=21866&oenc=1201&ieol=FF&oeol=PS")
             .waitForElementVisible("#rec-list li.operation");
 
         browser.expect.element(`#input-text .cm-content`).to.have.property("textContent").match(/^.{65}$/);