mirror of
https://github.com/lexborisov/Modest
synced 2024-11-21 21:31:25 +03:00
308 lines
14 KiB
C
308 lines
14 KiB
C
/*
|
|
Copyright (C) 2015-2017 Alexander Borisov
|
|
|
|
This library is free software; you can redistribute it and/or
|
|
modify it under the terms of the GNU Lesser General Public
|
|
License as published by the Free Software Foundation; either
|
|
version 2.1 of the License, or (at your option) any later version.
|
|
|
|
This library is distributed in the hope that it will be useful,
|
|
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
|
Lesser General Public License for more details.
|
|
|
|
You should have received a copy of the GNU Lesser General Public
|
|
License along with this library; if not, write to the Free Software
|
|
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
|
|
|
|
Author: lex.borisov@gmail.com (Alexander Borisov)
|
|
*/
|
|
|
|
#include "myhtml/parser.h"
|
|
|
|
void myhtml_parser_stream(mythread_id_t thread_id, void* ctx)
|
|
{
|
|
mythread_queue_node_t *qnode = (mythread_queue_node_t*)ctx;
|
|
|
|
if((((myhtml_tree_t*)(qnode->context))->parse_flags & MyHTML_TREE_PARSE_FLAGS_WITHOUT_BUILD_TREE) == 0) {
|
|
while(myhtml_rules_tree_dispatcher(qnode->context, qnode->args)){}
|
|
}
|
|
}
|
|
|
|
size_t myhtml_parser_token_data_to_string_lowercase(myhtml_tree_t *tree, mycore_string_t* str, myhtml_data_process_entry_t* proc_entry, size_t begin, size_t length)
|
|
{
|
|
mycore_incoming_buffer_t *buffer = mycore_incoming_buffer_find_by_position(tree->incoming_buf_first, begin);
|
|
size_t relative_begin = begin - buffer->offset;
|
|
|
|
// if token data length in one buffer then print them all at once
|
|
if((relative_begin + length) <= buffer->size) {
|
|
if(tree->encoding == MyENCODING_UTF_8)
|
|
myhtml_string_append_lowercase_with_preprocessing(str, &buffer->data[relative_begin], length, proc_entry->emit_null_char);
|
|
else
|
|
myhtml_string_append_lowercase_chunk_with_convert_encoding_with_preprocessing(str, &proc_entry->res,
|
|
&buffer->data[relative_begin], length,
|
|
proc_entry->encoding, proc_entry->emit_null_char);
|
|
|
|
return str->length;
|
|
}
|
|
|
|
size_t save_position = 0;
|
|
// if the data are spread across multiple buffers that join them
|
|
while(buffer) {
|
|
if((relative_begin + length) > buffer->size)
|
|
{
|
|
size_t relative_end = (buffer->size - relative_begin);
|
|
length -= relative_end;
|
|
|
|
size_t tmp_offset = myhtml_string_before_append_any_preprocessing(str, &buffer->data[relative_begin], relative_end, save_position);
|
|
|
|
if(relative_end > 0) {
|
|
if(tree->encoding == MyENCODING_UTF_8)
|
|
save_position = myhtml_string_append_lowercase_with_preprocessing(str, &buffer->data[(relative_begin + tmp_offset)], (relative_end - tmp_offset), proc_entry->emit_null_char);
|
|
else
|
|
save_position = myhtml_string_append_lowercase_chunk_with_convert_encoding_with_preprocessing(str, &proc_entry->res,
|
|
&buffer->data[(relative_begin + tmp_offset)], (relative_end - tmp_offset),
|
|
proc_entry->encoding, proc_entry->emit_null_char);
|
|
}
|
|
|
|
relative_begin = 0;
|
|
buffer = buffer->next;
|
|
}
|
|
else {
|
|
size_t tmp_offset = myhtml_string_before_append_any_preprocessing(str, &buffer->data[relative_begin], length, save_position);
|
|
|
|
if(length > 0) {
|
|
if(tree->encoding == MyENCODING_UTF_8)
|
|
myhtml_string_append_lowercase_with_preprocessing(str, &buffer->data[(relative_begin + tmp_offset)], (length - tmp_offset), proc_entry->emit_null_char);
|
|
else
|
|
myhtml_string_append_lowercase_chunk_with_convert_encoding_with_preprocessing(str, &proc_entry->res,
|
|
&buffer->data[(relative_begin + tmp_offset)], (length - tmp_offset),
|
|
proc_entry->encoding, proc_entry->emit_null_char);
|
|
}
|
|
|
|
break;
|
|
}
|
|
}
|
|
|
|
return str->length;
|
|
}
|
|
|
|
size_t myhtml_parser_token_data_to_string(myhtml_tree_t *tree, mycore_string_t* str, myhtml_data_process_entry_t* proc_entry, size_t begin, size_t length)
|
|
{
|
|
mycore_incoming_buffer_t *buffer = mycore_incoming_buffer_find_by_position(tree->incoming_buf_first, begin);
|
|
size_t relative_begin = begin - buffer->offset;
|
|
|
|
// if token data length in one buffer then print them all at once
|
|
if((relative_begin + length) <= buffer->size) {
|
|
if(tree->encoding == MyENCODING_UTF_8)
|
|
myhtml_string_append_with_preprocessing(str, &buffer->data[relative_begin], length, proc_entry->emit_null_char);
|
|
else
|
|
myhtml_string_append_chunk_with_convert_encoding_with_preprocessing(str, &proc_entry->res,
|
|
&buffer->data[relative_begin], length,
|
|
proc_entry->encoding, proc_entry->emit_null_char);
|
|
|
|
return str->length;
|
|
}
|
|
|
|
size_t save_position = 0;
|
|
// if the data are spread across multiple buffers that join them
|
|
while(buffer) {
|
|
if((relative_begin + length) > buffer->size)
|
|
{
|
|
size_t relative_end = (buffer->size - relative_begin);
|
|
length -= relative_end;
|
|
|
|
size_t tmp_offset = myhtml_string_before_append_any_preprocessing(str, &buffer->data[relative_begin], relative_end, save_position);
|
|
|
|
if(relative_end > 0) {
|
|
if(tree->encoding == MyENCODING_UTF_8)
|
|
save_position = myhtml_string_append_with_preprocessing(str, &buffer->data[(relative_begin + tmp_offset)], (relative_end - tmp_offset), proc_entry->emit_null_char);
|
|
else
|
|
save_position = myhtml_string_append_chunk_with_convert_encoding_with_preprocessing(str, &proc_entry->res,
|
|
&buffer->data[(relative_begin + tmp_offset)],
|
|
(relative_end - tmp_offset),
|
|
proc_entry->encoding, proc_entry->emit_null_char);
|
|
}
|
|
|
|
relative_begin = 0;
|
|
buffer = buffer->next;
|
|
}
|
|
else {
|
|
size_t tmp_offset = myhtml_string_before_append_any_preprocessing(str, &buffer->data[relative_begin], length, save_position);
|
|
|
|
if(length > 0) {
|
|
if(tree->encoding == MyENCODING_UTF_8)
|
|
myhtml_string_append_with_preprocessing(str, &buffer->data[(relative_begin + tmp_offset)], (length - tmp_offset), proc_entry->emit_null_char);
|
|
else
|
|
myhtml_string_append_chunk_with_convert_encoding_with_preprocessing(str, &proc_entry->res,
|
|
&buffer->data[(relative_begin + tmp_offset)], (length - tmp_offset),
|
|
proc_entry->encoding, proc_entry->emit_null_char);
|
|
}
|
|
|
|
break;
|
|
}
|
|
}
|
|
|
|
return str->length;
|
|
}
|
|
|
|
size_t myhtml_parser_token_data_to_string_charef(myhtml_tree_t *tree, mycore_string_t* str, myhtml_data_process_entry_t* proc_entry, size_t begin, size_t length)
|
|
{
|
|
mycore_incoming_buffer_t *buffer = mycore_incoming_buffer_find_by_position(tree->incoming_buf_first, begin);
|
|
size_t relative_begin = begin - buffer->offset;
|
|
|
|
// if token data length in one buffer then print them all at once
|
|
if((relative_begin + length) <= buffer->size) {
|
|
myhtml_data_process(proc_entry, str, &buffer->data[relative_begin], length);
|
|
myhtml_data_process_end(proc_entry, str);
|
|
|
|
return str->length;
|
|
}
|
|
|
|
// if the data are spread across multiple buffers that join them
|
|
while(buffer) {
|
|
if((relative_begin + length) > buffer->size)
|
|
{
|
|
size_t relative_end = (buffer->size - relative_begin);
|
|
length -= relative_end;
|
|
|
|
myhtml_data_process(proc_entry, str, &buffer->data[relative_begin], relative_end);
|
|
|
|
relative_begin = 0;
|
|
buffer = buffer->next;
|
|
}
|
|
else {
|
|
myhtml_data_process(proc_entry, str, &buffer->data[relative_begin], length);
|
|
break;
|
|
}
|
|
}
|
|
|
|
myhtml_data_process_end(proc_entry, str);
|
|
|
|
return str->length;
|
|
}
|
|
|
|
void myhtml_parser_worker(mythread_id_t thread_id, void* ctx)
|
|
{
|
|
mythread_queue_node_t *qnode = (mythread_queue_node_t*)ctx;
|
|
|
|
myhtml_tree_t* tree = qnode->context;
|
|
myhtml_token_node_t* token = qnode->args;
|
|
|
|
/*
|
|
* Tree can not be built without tokens
|
|
*
|
|
* MyHTML_TREE_PARSE_FLAGS_WITHOUT_PROCESS_TOKEN == 3
|
|
* MyHTML_TREE_PARSE_FLAGS_WITHOUT_BUILD_TREE == 1
|
|
*
|
|
* MyHTML_TREE_PARSE_FLAGS_WITHOUT_PROCESS_TOKEN include MyHTML_TREE_PARSE_FLAGS_WITHOUT_BUILD_TREE
|
|
*
|
|
* if set only MyHTML_TREE_PARSE_FLAGS_WITHOUT_BUILD_TREE and check only for MyHTML_TREE_PARSE_FLAGS_WITHOUT_PROCESS_TOKEN
|
|
* return true
|
|
* we need check both, 1 and 2
|
|
*/
|
|
if((tree->parse_flags & MyHTML_TREE_PARSE_FLAGS_WITHOUT_PROCESS_TOKEN) &&
|
|
(tree->parse_flags & 2))
|
|
{
|
|
if(tree->callback_before_token)
|
|
tree->callback_before_token_ctx = tree->callback_before_token(tree, token, tree->callback_before_token_ctx);
|
|
|
|
token->type |= MyHTML_TOKEN_TYPE_DONE;
|
|
|
|
if(tree->callback_after_token)
|
|
tree->callback_after_token_ctx = tree->callback_after_token(tree, token, tree->callback_after_token_ctx);
|
|
|
|
return;
|
|
}
|
|
|
|
size_t mchar_node_id;
|
|
#ifndef MyCORE_BUILD_WITHOUT_THREADS
|
|
if(tree->myhtml->thread_batch)
|
|
mchar_node_id = tree->async_args[(thread_id + tree->myhtml->thread_batch->id_increase)].mchar_node_id;
|
|
else
|
|
#endif
|
|
mchar_node_id = tree->async_args[thread_id].mchar_node_id;
|
|
|
|
if(tree->callback_before_token)
|
|
tree->callback_before_token_ctx = tree->callback_before_token(tree, token, tree->callback_before_token_ctx);
|
|
|
|
if(token->tag_id == MyHTML_TAG__TEXT ||
|
|
token->tag_id == MyHTML_TAG__COMMENT)
|
|
{
|
|
mycore_string_init(tree->mchar, mchar_node_id, &token->str, (token->raw_length + 1));
|
|
|
|
token->attr_first = NULL;
|
|
token->attr_last = NULL;
|
|
|
|
myhtml_data_process_entry_t proc_entry;
|
|
myhtml_data_process_entry_clean(&proc_entry);
|
|
|
|
proc_entry.encoding = tree->encoding;
|
|
|
|
if(token->type & MyHTML_TOKEN_TYPE_DATA) {
|
|
proc_entry.emit_null_char = true;
|
|
|
|
myhtml_parser_token_data_to_string_charef(tree, &token->str, &proc_entry, token->raw_begin, token->raw_length);
|
|
}
|
|
else if(token->type & MyHTML_TOKEN_TYPE_RCDATA || token->type & MyHTML_TOKEN_TYPE_CDATA) {
|
|
myhtml_parser_token_data_to_string_charef(tree, &token->str, &proc_entry, token->raw_begin, token->raw_length);
|
|
}
|
|
else
|
|
myhtml_parser_token_data_to_string(tree, &token->str, &proc_entry, token->raw_begin, token->raw_length);
|
|
}
|
|
else if(token->attr_first)
|
|
{
|
|
mycore_string_clean_all(&token->str);
|
|
|
|
myhtml_token_attr_t* attr = token->attr_first;
|
|
myhtml_data_process_entry_t proc_entry;
|
|
|
|
while(attr)
|
|
{
|
|
if(attr->raw_key_length) {
|
|
myhtml_data_process_entry_clean(&proc_entry);
|
|
proc_entry.encoding = tree->encoding;
|
|
|
|
mycore_string_init(tree->mchar, mchar_node_id, &attr->key, (attr->raw_key_length + 1));
|
|
myhtml_parser_token_data_to_string_lowercase(tree, &attr->key, &proc_entry, attr->raw_key_begin, attr->raw_key_length);
|
|
}
|
|
else
|
|
mycore_string_clean_all(&attr->key);
|
|
|
|
if(attr->raw_value_length) {
|
|
myhtml_data_process_entry_clean(&proc_entry);
|
|
proc_entry.encoding = tree->encoding;
|
|
proc_entry.is_attributes = true;
|
|
|
|
mycore_string_init(tree->mchar, mchar_node_id, &attr->value, (attr->raw_value_length + 1));
|
|
myhtml_parser_token_data_to_string_charef(tree, &attr->value, &proc_entry, attr->raw_value_begin, attr->raw_value_length);
|
|
}
|
|
else
|
|
mycore_string_clean_all(&attr->value);
|
|
|
|
attr = attr->next;
|
|
}
|
|
}
|
|
else {
|
|
token->attr_first = NULL;
|
|
token->attr_last = NULL;
|
|
|
|
mycore_string_clean_all(&token->str);
|
|
}
|
|
|
|
token->type |= MyHTML_TOKEN_TYPE_DONE;
|
|
|
|
if(tree->callback_after_token)
|
|
tree->callback_after_token_ctx = tree->callback_after_token(tree, token, tree->callback_after_token_ctx);
|
|
}
|
|
|
|
void myhtml_parser_worker_stream(mythread_id_t thread_id, void* ctx)
|
|
{
|
|
mythread_queue_node_t *qnode = (mythread_queue_node_t*)ctx;
|
|
|
|
myhtml_parser_worker(thread_id, qnode);
|
|
myhtml_parser_stream(thread_id, qnode);
|
|
}
|
|
|
|
|