src/blocks.c



#include <stdlib.h>
#include <assert.h>
#include <stdio.h>
#include <stdbool.h>
#include <ctype.h>

#include "stmd.h"
#include "utf8.h"
#include "html/houdini.h"
#include "scanners.h"
#include "uthash.h"

#define peek_at(i, n) (i)->data[n]

static void incorporate_line(strbuf *ln, int line_number, node_block** curptr);
static void finalize(node_block* b, int line_number);

static node_block* make_block(int tag, int start_line, int start_column)
{
    node_block* e;

    e = malloc(sizeof(node_block));
    memset(e, 0x0, sizeof(*e));

    e->tag = tag;
    e->open = true;
    e->start_line = start_line;
    e->start_column = start_column;
    e->end_line = start_line;
    strbuf_init(&e->string_content, 32);

    return e;
}

// Create a root document node_block.
extern node_block* make_document()
{
    node_block *e = make_block(BLOCK_DOCUMENT, 1, 1);
    reference *map = NULL;
    reference ** refmap;

    refmap = (reference**) malloc(sizeof(reference*));
    *refmap = map;
    e->as.document.refmap = refmap;
    e->top = e;

    return e;
}

// Returns true if line has only space characters, else false.
bool is_blank(strbuf *s, int offset)
{
    while (offset < s->size) {
        switch (s->ptr[offset]) {
            case '\n':
                return true;
            case ' ':
                offset++;
                break;
            default:
                return false;
        }
    }

    return true;
}

static inline bool can_contain(int parent_type, int child_type)
{
    return ( parent_type == BLOCK_DOCUMENT ||
            parent_type == BLOCK_BQUOTE ||
            parent_type == BLOCK_LIST_ITEM ||
            (parent_type == BLOCK_LIST && child_type == BLOCK_LIST_ITEM) );
}

static inline bool accepts_lines(int block_type)
{
    return (block_type == BLOCK_PARAGRAPH ||
            block_type == BLOCK_ATX_HEADER ||
            block_type == BLOCK_INDENTED_CODE ||
            block_type == BLOCK_FENCED_CODE);
}

static void add_line(node_block* node_block, chunk *ch, int offset)
{
    assert(node_block->open);
    strbuf_put(&node_block->string_content, ch->data + offset, ch->len - offset);
}

static void remove_trailing_blank_lines(strbuf *ln)
{
    int i;

    for (i = ln->size - 1; i >= 0; --i) {
        char c = ln->ptr[i];

        if (c != ' ' && c != '\t' && c != '\r' && c != '\n')
            break;
    }

    if (i < 0) {
        strbuf_clear(ln);
        return;
    }

    i = strbuf_strchr(ln, '\n', i);
    if (i >= 0)
        strbuf_truncate(ln, i);
}

// Check to see if a node_block ends with a blank line, descending
// if needed into lists and sublists.
static bool ends_with_blank_line(node_block* node_block)
{
    if (node_block->last_line_blank) {
        return true;
    }
    if ((node_block->tag == BLOCK_LIST || node_block->tag == BLOCK_LIST_ITEM) && node_block->last_child) {
        return ends_with_blank_line(node_block->last_child);
    } else {
        return false;
    }
}

// Break out of all containing lists
static int break_out_of_lists(node_block ** bptr, int line_number)
{
    node_block *container = *bptr;
    node_block *b = container->top;
    // find first containing BLOCK_LIST:
    while (b && b->tag != BLOCK_LIST) {
        b = b->last_child;
    }
    if (b) {
        while (container && container != b) {
            finalize(container, line_number);
            container = container->parent;
        }
        finalize(b, line_number);
        *bptr = b->parent;
    }
    return 0;
}


static void finalize(node_block* b, int line_number)
{
    int firstlinelen;
    int pos;
    node_block* item;
    node_block* subitem;

    if (!b->open)
        return; // don't do anything if the node_block is already closed

    b->open = false;
    if (line_number > b->start_line) {
        b->end_line = line_number - 1;
    } else {
        b->end_line = line_number;
    }

    switch (b->tag) {
        case BLOCK_PARAGRAPH:
            pos = 0;
            while (strbuf_at(&b->string_content, 0) == '[' &&
                    (pos = parse_reference(&b->string_content, b->top->as.document.refmap))) {

                strbuf_drop(&b->string_content, pos);
            }
            if (is_blank(&b->string_content, 0)) {
                b->tag = BLOCK_REFERENCE_DEF;
            }
            break;

        case BLOCK_INDENTED_CODE:
            remove_trailing_blank_lines(&b->string_content);
            strbuf_putc(&b->string_content, '\n');
            break;

        case BLOCK_FENCED_CODE:
            // first line of contents becomes info
            firstlinelen = strbuf_strchr(&b->string_content, '\n', 0);

            strbuf_init(&b->as.code.info, 0);
            houdini_unescape_html_f(
                &b->as.code.info,
                b->string_content.ptr,
                firstlinelen
            );

            strbuf_drop(&b->string_content, firstlinelen + 1);

            strbuf_trim(&b->as.code.info);
            unescape_buffer(&b->as.code.info);
            break;

        case BLOCK_LIST: // determine tight/loose status
            b->as.list.tight = true; // tight by default
            item = b->children;

            while (item) {
                // check for non-final non-empty list item ending with blank line:
                if (item->last_line_blank && item->next) {
                    b->as.list.tight = false;
                    break;
                }
                // recurse into children of list item, to see if there are
                // spaces between them:
                subitem = item->children;
                while (subitem) {
                    if (ends_with_blank_line(subitem) &&
                            (item->next || subitem->next)) {
                        b->as.list.tight = false;
                        break;
                    }
                    subitem = subitem->next;
                }
                if (!(b->as.list.tight)) {
                    break;
                }
                item = item->next;
            }

            break;

        default:
            break;
    }
}

// Add a node_block as child of another.  Return pointer to child.
extern node_block* add_child(node_block* parent,
        int block_type, int start_line, int start_column)
{
    assert(parent);

    // if 'parent' isn't the kind of node_block that can accept this child,
    // then back up til we hit a node_block that can.
    while (!can_contain(parent->tag, block_type)) {
        finalize(parent, start_line);
        parent = parent->parent;
    }

    node_block* child = make_block(block_type, start_line, start_column);
    child->parent = parent;
    child->top = parent->top;

    if (parent->last_child) {
        parent->last_child->next = child;
        child->prev = parent->last_child;
    } else {
        parent->children = child;
        child->prev = NULL;
    }
    parent->last_child = child;
    return child;
}

// Free a node_block list and any children.
extern void free_blocks(node_block* e)
{
    node_block * next;
    while (e != NULL) {
        next = e->next;
        free_inlines(e->inline_content);
        strbuf_free(&e->string_content);
        if (e->tag == BLOCK_FENCED_CODE) {
            strbuf_free(&e->as.code.info);
        } else if (e->tag == BLOCK_DOCUMENT) {
            free_reference_map(e->as.document.refmap);
        }
        free_blocks(e->children);
        free(e);
        e = next;
    }
}

// Walk through node_block and all children, recursively, parsing
// string content into inline content where appropriate.
void process_inlines(node_block* cur, reference** refmap)
{
    switch (cur->tag) {
        case BLOCK_PARAGRAPH:
        case BLOCK_ATX_HEADER:
        case BLOCK_SETEXT_HEADER:
            cur->inline_content = parse_inlines(&cur->string_content, refmap);
            // MEM
            // strbuf_free(&cur->string_content);
            break;

        default:
            break;
    }

    node_block *child = cur->children;
    while (child != NULL) {
        process_inlines(child, refmap);
        child = child->next;
    }
}

// Attempts to parse a list item marker (bullet or enumerated).
// On success, returns length of the marker, and populates
// data with the details.  On failure, returns 0.
static int parse_list_marker(chunk *input, int pos, struct ListData ** dataptr)
{
    unsigned char c;
    int startpos;
    struct ListData * data;

    startpos = pos;
    c = peek_at(input, pos);

    if ((c == '*' || c == '-' || c == '+') && !scan_hrule(input, pos)) {
        pos++;
        if (!isspace(peek_at(input, pos))) {
            return 0;
        }
        data = malloc(sizeof(struct ListData));
        data->marker_offset = 0; // will be adjusted later
        data->list_type = bullet;
        data->bullet_char = c;
        data->start = 1;
        data->delimiter = period;
        data->tight = false;

    } else if (isdigit(c)) {
        int start = 0;

        do {
            start = (10 * start) + (peek_at(input, pos) - '0');
            pos++;
        } while (isdigit(peek_at(input, pos)));

        c = peek_at(input, pos);
        if (c == '.' || c == ')') {
            pos++;
            if (!isspace(peek_at(input, pos))) {
                return 0;
            }
            data = malloc(sizeof(struct ListData));
            data->marker_offset = 0; // will be adjusted later
            data->list_type = ordered;
            data->bullet_char = 0;
            data->start = start;
            data->delimiter = (c == '.' ? period : parens);
            data->tight = false;
        } else {
            return 0;
        }

    } else {
        return 0;
    }

    *dataptr = data;
    return (pos - startpos);
}

// Return 1 if list item belongs in list, else 0.
static int lists_match(struct ListData *list_data, struct ListData *item_data)
{
    return (list_data->list_type == item_data->list_type &&
            list_data->delimiter == item_data->delimiter &&
            // list_data->marker_offset == item_data.marker_offset &&
            list_data->bullet_char == item_data->bullet_char);
}

static node_block *finalize_document(node_block *document, int linenum)
{
    while (document != document->top) {
        finalize(document, linenum);
        document = document->parent;
    }

    finalize(document, linenum);
    process_inlines(document, document->as.document.refmap);

    return document;
}

extern node_block *stmd_parse_file(FILE *f)
{
    strbuf line = GH_BUF_INIT;
    unsigned char buffer[4096];
    int linenum = 1;
    node_block *document = make_document();

    while (fgets((char *)buffer, sizeof(buffer), f)) {
        utf8proc_detab(&line, buffer, strlen((char *)buffer));
        incorporate_line(&line, linenum, &document);
        strbuf_clear(&line);
        linenum++;
    }

    strbuf_free(&line);
    return finalize_document(document, linenum);
}

extern node_block *stmd_parse_document(const unsigned char *buffer, size_t len)
{
    strbuf line = GH_BUF_INIT;
    int linenum = 1;
    const unsigned char *end = buffer + len;
    node_block *document = make_document();

    while (buffer < end) {
        const unsigned char *eol = memchr(buffer, '\n', end - buffer);

        if (!eol) {
            utf8proc_detab(&line, buffer, end - buffer);
            buffer = end;
        } else {
            utf8proc_detab(&line, buffer, (eol - buffer) + 1);
            buffer += (eol - buffer) + 1;
        }

        incorporate_line(&line, linenum, &document);
        strbuf_clear(&line);
        linenum++;
    }

    strbuf_free(&line);
    return finalize_document(document, linenum);
}

static void chop_trailing_hashtags(chunk *ch)
{
    int n;

    chunk_rtrim(ch);
    n = ch->len - 1;

    // if string ends in #s, remove these:
    while (n >= 0 && peek_at(ch, n) == '#')
        n--;

    // the last # was escaped, so we include it.
    if (n >= 0 && peek_at(ch, n) == '\\')
        n++;

    ch->len = n + 1;
}

// Process one line at a time, modifying a node_block.
static void incorporate_line(strbuf *line, int line_number, node_block** curptr)
{
    node_block* last_matched_container;
    int offset = 0;
    int matched = 0;
    int lev = 0;
    int i;
    struct ListData * data = NULL;
    bool all_matched = true;
    node_block* container;
    node_block* cur = *curptr;
    bool blank = false;
    int first_nonspace;
    int indent;
    chunk input;

    input.data = line->ptr;
    input.len = line->size;

    // container starts at the document root.
    container = cur->top;

    // for each containing node_block, try to parse the associated line start.
    // bail out on failure:  container will point to the last matching node_block.

    while (container->last_child && container->last_child->open) {
        container = container->last_child;

        first_nonspace = offset;
        while (peek_at(&input, first_nonspace) == ' ') {
            first_nonspace++;
        }

        indent = first_nonspace - offset;
        blank = peek_at(&input, first_nonspace) == '\n';

        if (container->tag == BLOCK_BQUOTE) {
            matched = indent <= 3 && peek_at(&input, first_nonspace) == '>';
            if (matched) {
                offset = first_nonspace + 1;
                if (peek_at(&input, offset) == ' ')
                    offset++;
            } else {
                all_matched = false;
            }

        } else if (container->tag == BLOCK_LIST_ITEM) {

            if (indent >= container->as.list.marker_offset +
                    container->as.list.padding) {
                offset += container->as.list.marker_offset +
                    container->as.list.padding;
            } else if (blank) {
                offset = first_nonspace;
            } else {
                all_matched = false;
            }

        } else if (container->tag == BLOCK_INDENTED_CODE) {

            if (indent >= CODE_INDENT) {
                offset += CODE_INDENT;
            } else if (blank) {
                offset = first_nonspace;
            } else {
                all_matched = false;
            }

        } else if (container->tag == BLOCK_ATX_HEADER ||
                container->tag == BLOCK_SETEXT_HEADER) {

            // a header can never contain more than one line
            all_matched = false;

        } else if (container->tag == BLOCK_FENCED_CODE) {

            // skip optional spaces of fence offset
            i = container->as.code.fence_offset;
            while (i > 0 && peek_at(&input, offset) == ' ') {
                offset++;
                i--;
            }

        } else if (container->tag == BLOCK_HTML) {

            if (blank) {
                all_matched = false;
            }

        } else if (container->tag == BLOCK_PARAGRAPH) {

            if (blank) {
                container->last_line_blank = true;
                all_matched = false;
            }

        }

        if (!all_matched) {
            container = container->parent;  // back up to last matching node_block
            break;
        }
    }

    last_matched_container = container;

    // check to see if we've hit 2nd blank line, break out of list:
    if (blank && container->last_line_blank) {
        break_out_of_lists(&container, line_number);
    }

    // unless last matched container is code node_block, try new container starts:
    while (container->tag != BLOCK_FENCED_CODE && container->tag != BLOCK_INDENTED_CODE &&
            container->tag != BLOCK_HTML) {

        first_nonspace = offset;
        while (peek_at(&input, first_nonspace) == ' ')
            first_nonspace++;

        indent = first_nonspace - offset;
        blank = peek_at(&input, first_nonspace) == '\n';

        if (indent >= CODE_INDENT) {
            if (cur->tag != BLOCK_PARAGRAPH && !blank) {
                offset += CODE_INDENT;
                container = add_child(container, BLOCK_INDENTED_CODE, line_number, offset + 1);
            } else { // indent > 4 in lazy line
                break;
            }

        } else if (peek_at(&input, first_nonspace) == '>') {

            offset = first_nonspace + 1;
            // optional following character
            if (peek_at(&input, offset) == ' ')
                offset++;
            container = add_child(container, BLOCK_BQUOTE, line_number, offset + 1);

        } else if ((matched = scan_atx_header_start(&input, first_nonspace))) {

            offset = first_nonspace + matched;
            container = add_child(container, BLOCK_ATX_HEADER, line_number, offset + 1);

            int hashpos = chunk_strchr(&input, '#', first_nonspace);
            int level = 0;

            while (peek_at(&input, hashpos) == '#') {
                level++;
                hashpos++;
            }
            container->as.header.level = level;

        } else if ((matched = scan_open_code_fence(&input, first_nonspace))) {

            container = add_child(container, BLOCK_FENCED_CODE, line_number, first_nonspace + 1);
            container->as.code.fence_char = peek_at(&input, first_nonspace);
            container->as.code.fence_length = matched;
            container->as.code.fence_offset = first_nonspace - offset;
            offset = first_nonspace + matched;

        } else if ((matched = scan_html_block_tag(&input, first_nonspace))) {

            container = add_child(container, BLOCK_HTML, line_number, first_nonspace + 1);
            // note, we don't adjust offset because the tag is part of the text

        } else if (container->tag == BLOCK_PARAGRAPH &&
                (lev = scan_setext_header_line(&input, first_nonspace)) &&
                // check that there is only one line in the paragraph:
                strbuf_strrchr(&container->string_content, '\n',
                    strbuf_len(&container->string_content) - 2) < 0) {

            container->tag = BLOCK_SETEXT_HEADER;
            container->as.header.level = lev;
            offset = input.len - 1;

        } else if (!(container->tag == BLOCK_PARAGRAPH && !all_matched) &&
                (matched = scan_hrule(&input, first_nonspace))) {

            // it's only now that we know the line is not part of a setext header:
            container = add_child(container, BLOCK_HRULE, line_number, first_nonspace + 1);
            finalize(container, line_number);
            container = container->parent;
            offset = input.len - 1;

        } else if ((matched = parse_list_marker(&input, first_nonspace, &data))) {

            // compute padding:
            offset = first_nonspace + matched;
            i = 0;
            while (i <= 5 && peek_at(&input, offset + i) == ' ') {
                i++;
            }
            // i = number of spaces after marker, up to 5
            if (i >= 5 || i < 1 || peek_at(&input, offset) == '\n') {
                data->padding = matched + 1;
                if (i > 0) {
                    offset += 1;
                }
            } else {
                data->padding = matched + i;
                offset += i;
            }

            // check container; if it's a list, see if this list item
            // can continue the list; otherwise, create a list container.

            data->marker_offset = indent;

            if (container->tag != BLOCK_LIST ||
                    !lists_match(&container->as.list, data)) {
                container = add_child(container, BLOCK_LIST, line_number,
                        first_nonspace + 1);

                memcpy(&container->as.list, data, sizeof(*data));
            }

            // add the list item
            container = add_child(container, BLOCK_LIST_ITEM, line_number,
                    first_nonspace + 1);
            /* TODO: static */
            memcpy(&container->as.list, data, sizeof(*data));
            free(data);
        } else {
            break;
        }

        if (accepts_lines(container->tag)) {
            // if it's a line container, it can't contain other containers
            break;
        }
    }

    // what remains at offset is a text line.  add the text to the
    // appropriate container.

    first_nonspace = offset;
    while (peek_at(&input, first_nonspace) == ' ')
        first_nonspace++;

    indent = first_nonspace - offset;
    blank = peek_at(&input, first_nonspace) == '\n';

    // node_block quote lines are never blank as they start with >
    // and we don't count blanks in fenced code for purposes of tight/loose
    // lists or breaking out of lists.  we also don't set last_line_blank
    // on an empty list item.
    container->last_line_blank = (blank &&
            container->tag != BLOCK_BQUOTE &&
            container->tag != BLOCK_FENCED_CODE &&
            !(container->tag == BLOCK_LIST_ITEM &&
                container->children == NULL &&
                container->start_line == line_number));

    node_block *cont = container;
    while (cont->parent) {
        cont->parent->last_line_blank = false;
        cont = cont->parent;
    }

    if (cur != last_matched_container &&
            container == last_matched_container &&
            !blank &&
            cur->tag == BLOCK_PARAGRAPH &&
            strbuf_len(&cur->string_content) > 0) {

        add_line(cur, &input, offset);

    } else { // not a lazy continuation

        // finalize any blocks that were not matched and set cur to container:
        while (cur != last_matched_container) {
            finalize(cur, line_number);
            cur = cur->parent;
            assert(cur != NULL);
        }

        if (container->tag == BLOCK_INDENTED_CODE) {

            add_line(container, &input, offset);

        } else if (container->tag == BLOCK_FENCED_CODE) {
            matched = 0;

            if (indent <= 3 &&
                peek_at(&input, first_nonspace) == container->as.code.fence_char) {
                int fence_len = scan_close_code_fence(&input, first_nonspace);
                if (fence_len > container->as.code.fence_length)
                    matched = 1;
            }

            if (matched) {
                // if closing fence, don't add line to container; instead, close it:
                finalize(container, line_number);
                container = container->parent; // back up to parent
            } else {
                add_line(container, &input, offset);
            }

        } else if (container->tag == BLOCK_HTML) {

            add_line(container, &input, offset);

        } else if (blank) {

            // ??? do nothing

        } else if (container->tag == BLOCK_ATX_HEADER) {

            chop_trailing_hashtags(&input);
            add_line(container, &input, first_nonspace);
            finalize(container, line_number);
            container = container->parent;

        } else if (accepts_lines(container->tag)) {

            add_line(container, &input, first_nonspace);

        } else if (container->tag != BLOCK_HRULE && container->tag != BLOCK_SETEXT_HEADER) {

            // create paragraph container for line
            container = add_child(container, BLOCK_PARAGRAPH, line_number, first_nonspace + 1);
            add_line(container, &input, first_nonspace);

        } else {
            assert(false);
        }

        *curptr = container;
    }
}