[Plugins Required] Fixing Disjoint Line Breaks in PDFs/When Copying Text

[RumiaKit]

Disclaimer

Is this project open source? Yes/No
Is this project completely free? Yes/No
Is this project made with AI beyond the author’s ability to comprehend how it works? Yes/No

---

Workflows

1. When using OCR technology like Window Powertools copying from screen.
2. When copying a YouTube transcript with line-breaks.
3. When copying from research papers, books, or other PDF files which have disjoint line breaks.
4. (etc.)

Showcase

A custom script which tries to “calculate” line breaks and groups text into sentences.

1. Paste the code snippet below into the User Plugins plugin.
2. TIP: Depending on your purposes, you can mess with the constants SENSITIVITY, SHORTER_THAN_AVG_RATIO, and AVG_LENGTH_PERCENT. These numbers work well for 60+ character lines.

/* Identifies "true line breaks" for resources which break every row */

const processText = (text) => {
    let normalized = text.replace(/\r\n/g, '\n').replace(/\r/g, '\n');
    const lines = normalized.split('\n').map(line => line.replace(/[ \t]+$/, ''));
    
    const outputLines = [];
    const lengthHistory = [];
    const HISTORY_SIZE = 6;
    const SENSITIVITY = 10;
    const SHORTER_THAN_AVG_RATIO = 1.1; 
    const AVG_LENGTH_PERCENT = 0.66; 

    // PART 1: Detect punctuation (only new bullet points at sentence-end)
    const PUNCTUATION_REGEX = /([.?!:'"\u201d]|\.{3})$/; 
    
    const calculateAverageLength = (history) => {
        if (history.length === 0) return 0;

        // PART 2.1: < "Average Length" Threshold = Irrelevant to "average"
        const maxLength = Math.max(...history); // Find (historical) max length
        const threshold = maxLength * AVG_LENGTH_PERCENT + SENSITIVITY; // Calc. threshold
        const relevantLengths = history.filter(len => len >= threshold); // Filter history
        if (relevantLengths.length === 0) {
            const total = history.reduce((sum, len) => sum + len, 0);
            return total / history.length;
        }

        // PART 2.2: Re-calculates new average w/ only relevant (longer) lines
        const total = relevantLengths.reduce((sum, len) => sum + len, 0);
        return total / relevantLengths.length;
    };

    for (let i = 0; i < lines.length; i++) {
        const currentLine = lines[i];
        const currentLength = currentLine.length;
        
        // PART 3.1: Identifies all "line vs. paragraph breaks"
        if (currentLength === 0) {
            if (outputLines.slice(-1)[0] !== '\n') { 
                outputLines.push('\n'); 
            }
            continue;
        }
        
        // PART 3.2: "Lookahead" -> Add 1/2 of next line's word
        let effectiveCurrentLength = currentLength;
        if (i < lines.length - 1) {
            const nextLine = lines[i + 1];
            if (nextLine.length > 0) {
                const firstWordMatch = nextLine.match(/^\S+/); // First word
                if (firstWordMatch) {
                    const firstWordLength = firstWordMatch[0].length;
                    effectiveCurrentLength += Math.floor(firstWordLength / 2); // Length += 1/2 next
                }
            }
        }

        // PART 3.3: If ends with punctuation & is ANOMOLY in short-length, add line break
        const endsWithPunctuation = PUNCTUATION_REGEX.test(currentLine);
        let shouldAddBreak = false;
        if (endsWithPunctuation) {
            const averageLength = calculateAverageLength(lengthHistory);
            let isAnomalouslyShort = false;
            if (lengthHistory.length >= 1) {  // Assumes 1st 2x lines aren't new paragraph
                if (effectiveCurrentLength < averageLength * SHORTER_THAN_AVG_RATIO) {
                    isAnomalouslyShort = true;
                }
            }
            if (isAnomalouslyShort || lengthHistory.length < 2) {
                shouldAddBreak = true;
            }
        }
        
        // PART 3.4: Update line length history
        if (currentLine.length > 0) {
             lengthHistory.push(currentLength);
             if (lengthHistory.length > HISTORY_SIZE) {
                 lengthHistory.shift();
             }
        }

        // PART 3.5: Splits if ANOMOLY short-length
        outputLines.push(currentLine); // Add current line to output

        // Add separator unless it's the last line
        if (i < lines.length - 1) {
            if (shouldAddBreak) { outputLines.push('\n'); } 
            else { outputLines.push(' '); }
        }
    }
    
    // PART 4: Final cleanup: remove potential double spaces and trim
    let finalOutput = outputLines.join('').replace(/ {2,}/g, ' ');
    return finalOutput.trim();
};

plugin.addCommand({
    name: 'Paste and Normalize Paragraphs',
    id: "paste-and-normalize-paragraphs",
    callback: async () => {
        const activeFile = app.workspace.getActiveFile();
        const activeView = app.workspace.activeLeaf.view;

        if (!activeView || !activeView.editor) {
            console.warn("No active editor view available.");
            return;
        }

        // PART 1: Normalize paragraphs
        const editor = activeView.editor;
        const currPos = editor.getCursor();

        const clipboard = await navigator.clipboard.readText();
        if (!clipboard) {
            console.warn("Clipboard is empty. Cannot paste.");
            return;
        }
        
        const processedText = processText(clipboard);
        editor.replaceRange(processedText, currPos);
        const endPos = editor.offsetToPos(editor.posToOffset(currPos) + processedText.length);
        editor.setSelection(currPos, endPos);
    }
});