gavindiaz/include
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity
Files
main
include/Marketeer/WebDataExtractor.mqh
T
amirghadiri1987 56b382609f Add files via upload
2025-02-07 19:12:13 +03:30

1629 lines
82 KiB
Plaintext
Raw Permalink Blame History

//+------------------------------------------------------------------+
//| WebDataExtractor.mqh |
//| Copyright (c) 2015-2019, Marketeer |
//| https://www.mql5.com/en/users/marketeer |
//| https://www.mql5.com/en/articles/5706/ |
//| https://www.mql5.com/en/articles/5913/ |
//| HTML to CSV converter |
//+------------------------------------------------------------------+
//#define HASHMAP_WARNING // HASHMAP_VERBOSE
#include <Marketeer/IndexMap.mqh>
#include <Marketeer/GroupSettings.mqh>
input GroupSettings HTMLR_Settings; // H T M L R E P O R T S E T T I N G S
input string RowSelector = "tr:has-n-children(13)[bgcolor^=\"#F\"]"; // · RowSelector
input string ColumnSettingsFile = "ReportHistoryDeals.cfg.csv"; // · ColumnSettingsFile
input string SubstitutionSettingsFile = ""; // · SubstitutionSettingsFile
input bool LogDomElements = false; // · LogDomElements
input bool LogDomWarnings = false; // · LogDomWarnings
input bool LogSelectedElements = false; // · LogSelectedElements
#define LOG_MASK_DOM 1
#define LOG_MASK_DOM_WARNING 2
#define LOG_MASK_SELECTOR 4
#define LOG_MASK_MAX 4
#define LOG_MASK_ALL ((LOG_MASK_MAX << 1) - 1)
ushort combinators[] =
{
' ', '+', '>', '~'
};
string empty_tags[] =
{
#include <Marketeer/empty_strings.h>
};
/*
CSS selectors:
1 * any element
1 .intro class="intro"
1 #id1 id="id1"
1 p p tag
1 div p p tag somewhere inside div tag
2 div > p p tag with parent div tag
2 div + p p tag as a sibling immediately after div tag, common parent
3 div ~ p p tag as a sibling somewhere after div tag (less strict than +), common parent
2 [target] a tag with target attribute
2 [target=_blank] a tag with attribute target="_blank"
2- [title~=flower] a tag with attribute title containing "space"-separated word "flower" somewhere
3 [title*=flower] a tag with attribute title containing "flower" sequence somewhere (less strict than ~)
2- [data-value|="foo"] a tag whose attribute value has this "foo" in a dash-separated list somewhere
3 [data-value^="foo"] attribute value starts with this
3 [data-value$="foo"] attribute value ends with this
2 p:first-child a tag <p> that is the first child of its parent
3 p:last-child a tag <p> that is the last child of its parent
3 p:nth-child(n)
3 p:nth-last-child(n)
3- p:not(selector)
http://www.w3.org/TR/CSS2/selector.html
A simple selector is either a type selector or universal selector followed immediately by
zero or more attribute selectors, ID selectors, or pseudo-classes, in any order.
The simple selector matches if all of its components match.
A selector is a chain of one or more simple selectors separated by combinators.
It always begins with a type selector or a universal selector. No other type selector or universal selector is allowed in the sequence.
Combinators are: white space, ">", and "+" (and "~" css3)
*/
enum StateBit
{
blank,
insideTagOpen,
insideTagClose,
insideComment,
insideScript
};
enum AttrBit
{
name,
value
};
class AttributesParser
{
private:
AttrBit state;
int offset;
int cursor;
int length;
string key;
void skipWhiteSpace(const string &data)
{
int n = StringLen(data);
while(offset < n && StringGetCharacter(data, offset) <= 32)
{
offset++;
cursor++;
}
}
public:
AttributesParser(): state(AttrBit::name), offset(0), cursor(0), key(""){}
void parseAll(const string &data, IndexMap &attributes)
{
length = StringLen(data);
while(parse(data, attributes));
if(offset < length)
{
string ending = StringSubstr(data, cursor);
StringTrimRight(ending);
StringTrimLeft(ending);
if(StringLen(ending) > 0)
{
attributes.set(ending);
}
else if(key != "")
{
attributes.set(key);
}
}
else if(key != "")
{
attributes.set(key);
}
}
bool parse(const string &data, IndexMap &attributes)
{
skipWhiteSpace(data);
int x = 0;
if(state == AttrBit::name)
{
while(offset < length)
{
x = StringGetCharacter(data, offset);
if(x == '=' || (x <= 32)) break;
offset++;
}
string attr = StringSubstr(data, cursor, offset - cursor);
if(StringLen(attr) > 0)
{
key = attr;
}
if(x == '=')
{
state = AttrBit::value;
offset++;
cursor = offset;
}
// else - attribute without value
return(offset < length);
}
else
if(state == AttrBit::value)
{
if(key == "")
{
Print("Wrong state::value with empty key");
}
ushort c = StringGetCharacter(data, offset);
if(c == '"' || c == '\'')
{
offset++;
cursor++;
while(offset < length)
{
if(StringGetCharacter(data, offset) == c) break;
offset++;
}
attributes.setValue(key, StringSubstr(data, cursor, offset - cursor));
key = "";
state = AttrBit::name;
offset++;
cursor = offset;
return(offset < length);
}
else
if(c > 32)
{
while(offset < length)
{
if(StringGetCharacter(data, offset) == ' ') break;
offset++;
}
attributes.setValue(key, StringSubstr(data, cursor, offset - cursor));
key = "";
state = AttrBit::name;
offset++;
cursor = offset;
return(offset < length);
}
else
if(c <= 32) // empty value, 'attr='
{
while(offset < length)
{
if(StringGetCharacter(data, offset) > ' ') break;
offset++;
}
attributes.set(key);
key = "";
state = AttrBit::name;
cursor = offset;
return(offset < length);
}
}
return(false);
}
};
enum PseudoClassModifier
{
none,
firstChild,
lastChild,
nthChild,
nthLastChild,
hasNthChildren
};
class SubSelector
{
public:
ushort type;
string value;
PseudoClassModifier modifier;
string param;
SubSelector(ushort t, string v): type(t), value(v), modifier(PseudoClassModifier::none) {}
SubSelector(ushort t, string v, PseudoClassModifier m): type(t), value(v), modifier(m) {}
SubSelector(ushort t, string v, PseudoClassModifier m, string p): type(t), value(v), modifier(m), param(p) {}
};
class SubSelectorArray
{
private:
SubSelector *selectors[];
IndexMap mod;
static TypeContainer<PseudoClassModifier> first;
static TypeContainer<PseudoClassModifier> last;
static TypeContainer<PseudoClassModifier> nth;
static TypeContainer<PseudoClassModifier> nthLast;
static TypeContainer<PseudoClassModifier> hasN;
void init()
{
mod.add(":first-child", &first);
mod.add(":last-child", &last);
mod.add(":nth-child", &nth);
mod.add(":nth-last-child", &nthLast);
mod.add(":has-n-children", &hasN);
}
void createFromString(const string &selector)
{
ushort p = 0; // previous/pending type
int ppos = 0;
int i, n = StringLen(selector);
bool quotes = false;
for(i = 0; i < n; i++)
{
ushort t = StringGetCharacter(selector, i);
if(t == '"')
{
quotes = !quotes;
}
if(quotes) continue;
if(t == '.' || t == '#' || t == '[' || t == ']')
{
string v = StringSubstr(selector, ppos, i - ppos);
if(i == 0) v = "*";
if(p == '[' && StringLen(v) > 0 && StringGetCharacter(v, StringLen(v) - 1) == ']')
{
v = StringSubstr(v, 0, StringLen(v) - 1);
}
add(p, v);
p = t;
if(p == ']') p = 0;
ppos = i + 1;
}
}
if(ppos < n)
{
string v = StringSubstr(selector, ppos, n - ppos);
if(p == '[' && StringLen(v) > 0 && StringGetCharacter(v, StringLen(v) - 1) == ']')
{
v = StringSubstr(v, 0, StringLen(v) - 1);
}
add(p, v);
}
}
public:
SubSelectorArray()
{
init();
}
SubSelectorArray(const string selector)
{
init();
createFromString(selector);
}
int size() const
{
return ArraySize(selectors);
}
SubSelector *operator[](int i) const
{
return selectors[i];
}
void add(const ushort t, string v)
{
int n = ArraySize(selectors);
ArrayResize(selectors, n + 1);
PseudoClassModifier m = PseudoClassModifier::none;
string param;
for(int j = 0; j < mod.getSize(); j++)
{
int p = StringFind(v, mod.getKey(j));
if(p > -1)
{
if(p + StringLen(mod.getKey(j)) < StringLen(v))
{
param = StringSubstr(v, p + StringLen(mod.getKey(j)));
if(StringGetCharacter(param, 0) == '(' && StringGetCharacter(param, StringLen(param) - 1) == ')')
{
param = StringSubstr(param, 1, StringLen(param) - 2);
}
else
{
param = "";
}
}
m = mod[j].get<PseudoClassModifier>();
v = StringSubstr(v, 0, p);
break;
}
}
if(t == '[' && m == PseudoClassModifier::none)
{
AttributesParser p;
IndexMap attr;
p.parseAll(v, attr);
// attributes are selected one by one: element[attr1=value][attr2=value]
// the map should contain only 1 valid pair at a time
if(attr.getSize() > 0)
{
param = attr.getKey(0);
v = attr[0] != NULL ? attr[0].get<string>() : "";
}
}
if(StringLen(param) == 0)
{
selectors[n] = new SubSelector(t, v, m);
}
else
{
selectors[n] = new SubSelector(t, v, m, param);
}
}
~SubSelectorArray()
{
int i, n = ArraySize(selectors);
for(i = 0; i < n; i++)
{
delete(selectors[i]);
}
}
};
TypeContainer<PseudoClassModifier> SubSelectorArray::first(PseudoClassModifier::firstChild);
TypeContainer<PseudoClassModifier> SubSelectorArray::last(PseudoClassModifier::lastChild);
TypeContainer<PseudoClassModifier> SubSelectorArray::nth(PseudoClassModifier::nthChild);
TypeContainer<PseudoClassModifier> SubSelectorArray::nthLast(PseudoClassModifier::nthLastChild);
TypeContainer<PseudoClassModifier> SubSelectorArray::hasN(PseudoClassModifier::hasNthChildren);
class DomIterator;
class DomElement
{
private:
string name;
string content;
IndexMap attributes;
DomElement *parent;
int level;
protected:
bool childrenOwner;
DomElement *children[];
void clear()
{
int i, n = ArraySize(children);
for(i = 0; i < n; i++)
{
delete children[i];
}
ArrayResize(children, 0);
}
bool isCombinator(ushort c)
{
for(int i = 0; i < ArraySize(combinators); i++)
{
if(combinators[i] == c) return(true);
}
return(false);
}
public:
DomElement(): parent(NULL), childrenOwner(true) {}
DomElement(const string n): parent(NULL), childrenOwner(true)
{
name = n;
}
DomElement(const string n, const string text): parent(NULL), childrenOwner(true)
{
name = n;
content = text;
}
DomElement(DomElement *p, const string &n, const string text = ""): childrenOwner(true)
{
p.addChild(&this);
parent = p;
level = p.level + 1;
name = n;
if(text != "") content = text;
}
~DomElement()
{
if(childrenOwner)
{
clear();
}
}
void addChild(DomElement *child)
{
int n = ArraySize(children);
ArrayResize(children, n + 1);
children[n] = child;
}
int getChildrenCount() const
{
return ArraySize(children);
}
DomElement *getChild(const int i) const
{
if(i >= 0 && i < ArraySize(children))
{
return children[i];
}
return NULL;
}
void addChildren(DomElement *p)
{
if(CheckPointer(p) == POINTER_DYNAMIC)
{
for(int i = 0; i < p.getChildrenCount(); i++)
{
addChild(p.getChild(i));
}
}
}
int getChildIndex(DomElement *e) const
{
for(int i = 0; i < ArraySize(children); i++)
{
if(children[i] == e) return(i);
}
return(-1);
}
void setName(string n)
{
name = n;
}
void setText(string t)
{
content = t;
}
DomElement *getParent() const
{
return parent;
}
string getName() const
{
return name;
}
string getText() const
{
string text;
for(int i = 0; i < ArraySize(children); i++)
{
text += children[i].getText();
}
return content + text;
}
int getLevel() const
{
return level;
}
void print(bool full = true)
{
PrintFormat("%" + IntegerToString(level) + "c %s, %s", ' ', name, attributes.asString());
if(full)
{
int i, n = ArraySize(children);
for(i = 0; i < n; i++)
{
children[i].print();
}
}
}
void printWithContent(bool full = true)
{
PrintFormat("%" + IntegerToString(level) + "c %s, %s, %s", ' ', name, content, attributes.asString());
if(full)
{
int i, n = ArraySize(children);
for(i = 0; i < n; i++)
{
children[i].printWithContent();
}
}
}
void printShort()
{
string id = "";
Container *pid = attributes["id"];
if(pid != NULL) id = pid.get<string>();
PrintFormat("%" + IntegerToString(level) + "c %d %s %s", ' ', level, name, id);
}
void printShortWithChildren()
{
string id = "";
Container *pid = attributes["id"];
if(pid != NULL) id = pid.get<string>();
PrintFormat("%" + IntegerToString(level) + "c %d %s %s", ' ', level, name, id);
int i, n = ArraySize(children);
for(i = 0; i < n; i++)
{
children[i].printShort();
}
}
void parseAttributes(const string &data)
{
AttributesParser p;
p.parseAll(data, attributes);
}
void setAttribute(const string id, Container *value)
{
attributes.set(id, value);
}
bool hasAttribute(const string key) const
{
return (attributes.isKeyExisting(key));
}
string getAttribute(const string key) const
{
Container *c = attributes[key];
if(c != NULL) return c.get<string>();
return "";
}
string getAttribute(const int index) const
{
Container *c = attributes[index];
if(c != NULL) return c.get<string>();
return "";
}
bool match(const SubSelectorArray *u)
{
bool matched = true;
int i, n = u.size();
for(i = 0; i < n && matched; i++)
{
if(u[i].type == 0) // tag name
{
if(u[i].value == "*")
{
// any tag
}
else
if(StringLen(u[i].value) > 0 && StringCompare(name, u[i].value) != 0)
{
matched = false;
}
else
if(u[i].modifier == PseudoClassModifier::firstChild)
{
if(parent != NULL && parent.getChildIndex(&this) != 0)
{
matched = false;
}
}
else
if(u[i].modifier == PseudoClassModifier::lastChild)
{
if(parent != NULL && parent.getChildIndex(&this) != parent.getChildrenCount() - 1)
{
matched = false;
}
}
else
if(u[i].modifier == PseudoClassModifier::nthChild)
{
int x = (int)StringToInteger(u[i].param);
if(parent != NULL && parent.getChildIndex(&this) != x - 1) // children are counted starting from 1
{
matched = false;
}
}
else
if(u[i].modifier == PseudoClassModifier::nthLastChild)
{
int x = (int)StringToInteger(u[i].param);
if(parent != NULL && parent.getChildrenCount() - parent.getChildIndex(&this) - 1 != x - 1)
{
matched = false;
}
}
else
if(u[i].modifier == PseudoClassModifier::hasNthChildren)
{
int x = (int)StringToInteger(u[i].param);
if(getChildrenCount() != x)
{
matched = false;
}
}
}
else
if(u[i].type == '.') // class
{
if(attributes.isKeyExisting("class"))
{
Container *c = attributes["class"];
if(c == NULL || StringFind(" " + c.get<string>() + " ", " " + u[i].value + " ") == -1)
{
matched = false;
}
}
else
{
matched = false;
}
}
else
if(u[i].type == '#') // id
{
if(attributes.isKeyExisting("id"))
{
Container *c = attributes["id"];
if(c == NULL || StringCompare(c.get<string>(), u[i].value) != 0)
{
matched = false;
}
}
else
{
matched = false;
}
}
else
if(u[i].type == '[') // attributes
{
string key = u[i].param;
string v = u[i].value;
ushort suffix = StringGetCharacter(key, StringLen(key) - 1);
if(suffix == '*' || suffix == '^' || suffix == '$') // contains, starts with, or ends with
{
key = StringSubstr(key, 0, StringLen(key) - 1);
}
else
{
suffix = 0;
}
if(hasAttribute(key) && attributes[key] != NULL)
{
if(StringLen(v) > 0)
{
if(suffix == 0)
{
if(key == "class")
{
matched &= (StringFind(" " + attributes[key].get<string>() + " ", " " + v + " ") > -1);
}
else
{
matched &= (StringCompare(v, attributes[key].get<string>()) == 0);
}
}
else
if(suffix == '*')
{
matched &= (StringFind(attributes[key].get<string>(), v) != -1);
}
else
if(suffix == '^')
{
matched &= (StringFind(attributes[key].get<string>(), v) == 0);
}
else
if(suffix == '$')
{
string x = attributes[key].get<string>();
if(StringLen(x) > StringLen(v))
{
matched &= (StringFind(x, v, StringLen(x) - StringLen(v)) == StringLen(v));
}
}
}
}
else
{
matched = false;
}
}
}
return matched;
}
bool find(const ushort op, const SubSelectorArray *selectors, DomIterator *output)
{
bool found = false;
int i, n;
if(op == ' ' || op == '>' || op == '/')
{
n = ArraySize(children);
for(i = 0; i < n; i++)
{
if(children[i].match(selectors))
{
if(op == '/')
{
found = true;
output.addChild(GetPointer(children[i]));
}
else
if(op == ' ')
{
DomElement *p = &this;
while(p != NULL)
{
if(output.getChildIndex(p) != -1)
{
found = true;
output.addChild(GetPointer(children[i]));
break;
}
p = p.parent;
}
}
else // op == '>'
{
if(output.getChildIndex(&this) != -1)
{
found = true;
output.addChild(GetPointer(children[i]));
}
}
}
children[i].find(op, selectors, output);
}
}
else
if(op == '+' || op == '~')
{
if(CheckPointer(parent) == POINTER_DYNAMIC)
{
if(output.getChildIndex(&this) != -1)
{
int q = parent.getChildIndex(&this);
if(q != -1)
{
n = (op == '+') ? (q + 2) : parent.getChildrenCount();
if(n > parent.getChildrenCount()) n = parent.getChildrenCount();
for(i = q + 1; i < n; i++)
{
DomElement *m = parent.getChild(i);
if(CheckPointer(m) != POINTER_DYNAMIC)
{
Print("bad:", name, " i=", i, " q=", q, " n=", n);
}
if(m.match(selectors))
{
found = true;
output.addChild(m);
}
}
}
else
{
Print("Error: can't find 'this' element");
}
}
}
else
{
if(name != "root")
{
Print("No parent for: ", name);
}
}
for(i = 0; i < ArraySize(children); i++)
{
found = children[i].find(op, selectors, output) || found;
}
}
else
{
Print("Error: unknown combinator:", ShortToString(op));
}
return found;
}
DomIterator *querySelect(const string q)
{
DomIterator *result = new DomIterator();
if(q == ".")
{
result.addChild(&this); // root
return result;
}
int cursor = 0; // where selector string started
int i, n = StringLen(q);
ushort p = 0; // previous character
ushort a = 0; // next/pending operator
ushort b = '/'; // current operator, root notation from the start
string selector = "*"; // current simple selector, 'any' by default
int index = 0; // position in the resulting array of objects
for(i = 0; i < n; i++)
{
ushort c = StringGetCharacter(q, i);
if(isCombinator(c))
{
a = c;
if(!isCombinator(p))
{
selector = StringSubstr(q, cursor, i - cursor);
}
else
{
// suppress blanks around other combinators
a = MathMax(c, p);
}
cursor = i + 1;
}
else
{
if(isCombinator(p))
{
index = result.getChildrenCount();
SubSelectorArray selectors(selector);
find(b, &selectors, result);
b = a;
// now we can delete outdated results in positions up to 'index'
result.removeFirst(index);
}
}
p = c;
}
if(cursor < i)
{
selector = StringSubstr(q, cursor, i - cursor);
index = result.getChildrenCount();
SubSelectorArray selectors(selector);
find(b, &selectors, result);
result.removeFirst(index);
}
return result;
}
IndexMap *tableSelect(const string rowSelector, const string &headers[], const string &columSelectors[], const string &dataSelectors[], const IndexMap *subst = NULL, const bool numericKeys = false)
{
if(ArraySize(columSelectors) != ArraySize(dataSelectors)) return NULL;
int n = ArraySize(columSelectors);
DomIterator *r = querySelect(rowSelector);
if((HtmlParser::isDebug() & LOG_MASK_SELECTOR) != 0)
{
r.printAllWithContent();
}
IndexMap *data = new IndexMap('\n');
int counter = 0;
r.rewind();
while(r.hasNext())
{
DomElement *e = r.next();
if((HtmlParser::isDebug() & LOG_MASK_SELECTOR) != 0)
{
Print("row N" + (string)counter);
}
string id = IntegerToString(counter);
if(!numericKeys) // we can store IDs of found elements in results, optionally
{
if(e.hasAttribute("id")) id += "-" + e.getAttribute("id");
}
IndexMap *row = new IndexMap();
for(int i = 0; i < n; i++)
{
if(StringLen(columSelectors[i]))
{
DomIterator *d = e.querySelect(columSelectors[i]);
if((HtmlParser::isDebug() & LOG_MASK_SELECTOR) != 0)
{
d.printAllWithContent();
}
string value;
if(d.getChildrenCount() > 0)
{
if(d.getChildrenCount() > 1)
{
if((HtmlParser::isDebug() & LOG_MASK_SELECTOR) != 0)
{
Print("Too many elements selected:", d.getChildrenCount());
}
}
if(dataSelectors[i] == "")
{
value = d[0].getText();
}
else
{
value = d[0].getAttribute(dataSelectors[i]);
}
if(CheckPointer(subst) != POINTER_INVALID)
{
if(subst.isKeyExisting(columSelectors[i]))
{
IndexMap *rules = dynamic_cast<IndexMap *>(subst[columSelectors[i]]);
if(rules != NULL)
{
for(int j = 0; j < rules.getSize(); j++)
{
StringReplace(value, rules.getKey(j), (rules[j] != NULL ? rules[j].asString() : ""));
}
}
}
}
StringTrimLeft(value);
StringTrimRight(value);
if(numericKeys)
{
row.setValue(IntegerToString(i), value);
}
else
{
row.setValue(headers[i]/*columSelectors[i] + dataSelectors[i]*/, value);
}
}
else // field not found
{
if(numericKeys)
{
row.set(IntegerToString(i));
}
else
{
row.set(columSelectors[i] + dataSelectors[i]);
}
}
delete d;
}
else // constant data
{
if(numericKeys)
{
row.setValue(IntegerToString(i), dataSelectors[i]);
}
else
{
row.setValue(dataSelectors[i], dataSelectors[i]);
}
}
}
if(row.getSize() > 0)
{
data.set(id, row);
counter++;
}
else
{
delete row;
}
}
delete r;
return data;
}
};
class DomIterator: public DomElement
{
private:
int cursor;
public:
DomIterator()
{
childrenOwner = false;
}
bool hasNext()
{
return(cursor < ArraySize(children));
}
DomElement *next()
{
if(hasNext())
{
return children[cursor++];
}
return NULL;
}
DomElement *operator[](int index)
{
if(index >= 0 && index < ArraySize(children))
{
return children[index];
}
return NULL;
}
void rewind()
{
cursor = 0;
}
void removeFirst(const int n)
{
int m = ArraySize(children);
if(n < m)
{
ArrayCopy(children, children, 0, n);
ArrayResize(children, m - n);
}
else
if(n == m)
{
ArrayResize(children, 0);
}
}
void printAll()
{
rewind();
while(hasNext())
{
DomElement *e = next();
e.print(false);
}
}
void printAllWithContent()
{
rewind();
while(hasNext())
{
DomElement *e = next();
e.printWithContent(false);
}
}
};
class HtmlParser
{
private:
const string TAG_OPEN_START;
const string TAG_OPEN_STOP;
const string TAG_OPENCLOSE_STOP;
const string TAG_CLOSE_START;
const string TAG_CLOSE_STOP;
const string COMMENT_START;
const string COMMENT_STOP;
const string SCRIPT_STOP;
StateBit state;
DomElement *root;
DomElement *cursor;
int offset;
IndexMap empties;
static int debugLevel;
public:
HtmlParser():
TAG_OPEN_START("<"),
TAG_OPEN_STOP(">"),
TAG_OPENCLOSE_STOP("/>"),
TAG_CLOSE_START("</"),
TAG_CLOSE_STOP(">"),
COMMENT_START("<!--"),
COMMENT_STOP("-->"),
SCRIPT_STOP("/script>"),
state(blank)
{
for(int i = 0; i < ArraySize(empty_tags); i++)
{
empties.set(empty_tags[i]);
}
}
~HtmlParser()
{
if(root != NULL)
{
delete root;
}
}
DomElement *parse(const string &html)
{
if(root != NULL)
{
delete root;
}
root = new DomElement("root");
cursor = root;
offset = 0;
while(processText(html));
return root;
}
bool processText(const string &html)
{
int p;
if(state == blank)
{
p = StringFind(html, "<", offset);
if(p == -1) // no more tags
{
return(false);
}
else if(p > 0)
{
if(p > offset)
{
string text = StringSubstr(html, offset, p - offset);
StringTrimLeft(text);
StringTrimRight(text);
StringReplace(text, "&nbsp;", "");
if(StringLen(text) > 0)
{
cursor.setText(text);
}
}
}
offset = p;
if(IsString(html, COMMENT_START)) state = insideComment;
else
if(IsString(html, TAG_CLOSE_START)) state = insideTagClose;
else
if(IsString(html, TAG_OPEN_START)) state = insideTagOpen;
return(true);
}
else
if(state == insideTagOpen)
{
offset++;
int pspace = StringFind(html, " ", offset);
int pright = StringFind(html, ">", offset);
p = MathMin(pspace, pright);
if(p == -1)
{
p = MathMax(pspace, pright);
}
if(p == -1 || pright == -1) // no tag closing
{
return(false);
}
if(pspace > pright)
{
pspace = -1; // outer space
}
bool selfclose = false;
if(IsString(html, TAG_OPENCLOSE_STOP, pright - StringLen(TAG_OPENCLOSE_STOP) + 1))
{
selfclose = true;
if(p == pright) p--;
pright--;
}
string name = StringSubstr(html, offset, p - offset);
StringToLower(name);
StringTrimRight(name);
DomElement *e = new DomElement(cursor, name);
if(pspace != -1)
{
string txt;
if(pright - pspace > 1)
{
txt = StringSubstr(html, pspace + 1, pright - (pspace + 1));
e.parseAttributes(txt);
}
}
if((HtmlParser::isDebug() & LOG_MASK_DOM) != 0)
{
e.print(false);
}
bool softSelfClose = false;
if(!selfclose)
{
if(empties.isKeyExisting(name))
{
selfclose = true;
softSelfClose = true;
}
}
pright++;
if(!selfclose)
{
cursor = e;
}
else
{
if(!softSelfClose) pright++;
}
offset = pright;
if((name == "script") && !selfclose)
{
state = insideScript;
}
else
{
state = blank;
}
return(true);
}
else
if(state == insideTagClose)
{
offset += StringLen(TAG_CLOSE_START);
p = StringFind(html, ">", offset);
if(p == -1)
{
return(false);
}
string tag = StringSubstr(html, offset, p - offset);
StringToLower(tag);
DomElement *rewind = cursor;
while(StringCompare(cursor.getName(), tag) != 0)
{
string previous = cursor.getName();
cursor = cursor.getParent();
if(cursor == NULL)
{
// orphan closing tag
cursor = rewind;
state = blank;
offset = p + 1;
return(true);
}
if((isDebug() & LOG_MASK_DOM_WARNING) != 0)
{
Print("Misplaced /", tag, ">. Go from ", previous, " up to ", cursor.getName(), " ", cursor.getLevel());
}
}
cursor = cursor.getParent();
if(cursor == NULL) return(false);
state = blank;
offset = p + 1;
return(true);
}
else
if(state == insideComment)
{
offset += StringLen(COMMENT_START);
p = StringFind(html, COMMENT_STOP, offset);
if(p == -1)
{
return(false);
}
offset = p + StringLen(COMMENT_STOP);
state = blank;
return(true);
}
else
if(state == insideScript)
{
p = StringFind(html, SCRIPT_STOP, offset);
if(p == -1)
{
return(false);
}
offset = p + StringLen(SCRIPT_STOP);
state = blank;
cursor = cursor.getParent();
if(cursor == NULL) return(false);
return(true);
}
return(false);
}
bool IsString(const string &html, const string x, int subset = -1)
{
if(subset == -1) subset = offset;
return(StringSubstr(html, subset, StringLen(x)) == x);
}
static void enableDebug(int level)
{
debugLevel = level;
}
static int isDebug()
{
return debugLevel;
}
};
int HtmlParser::debugLevel = 0;
class HTMLConverter
{
protected:
static bool loadColumnConfig(string &columnSelectors[], string &dataSelectors[], string &headers[])
{
Print("Reading column configuration ", ColumnSettingsFile);
int h = FileOpen(ColumnSettingsFile, FILE_READ|FILE_ANSI|FILE_TXT|FILE_SHARE_READ|FILE_SHARE_WRITE, '|', CP_UTF8);
if(h == -1)
{
Print("Error reading file '", ColumnSettingsFile, "': ", GetLastError());
return false;
}
int n = 0;
bool headerRead = false;
while(!FileIsEnding(h))
{
string stParts[];
string stLine = FileReadString(h);
int nParts = StringSplit(stLine, ',', stParts);
if(nParts == 3)
{
if(!headerRead)
{
headerRead = true;
continue;
}
ArrayResize(columnSelectors, n + 1);
ArrayResize(dataSelectors, n + 1);
ArrayResize(headers, n + 1);
headers[n] = stParts[0];
columnSelectors[n] = stParts[1];
dataSelectors[n] = stParts[2];
// Print("Column ", (n + 1), ": '", stParts[0], "', selector: '", stParts[1], "', locator: '", stParts[2], "'");
n++;
}
}
FileClose(h);
return true;
}
static bool loadSubstConfig(int &rulesColumns[], string &rulesHave[], string &rulesWant[])
{
Print("Reading column configuration ", SubstitutionSettingsFile);
int h = FileOpen(SubstitutionSettingsFile, FILE_READ|FILE_ANSI|FILE_TXT|FILE_SHARE_READ|FILE_SHARE_WRITE, '|', CP_UTF8);
if(h == -1)
{
Print("Error reading file '", SubstitutionSettingsFile, "': ", GetLastError());
return false;
}
int n = 0;
bool headerRead = false;
while(!FileIsEnding(h))
{
string stParts[];
string stLine = FileReadString(h);
int nParts = StringSplit(stLine, ',', stParts);
if(nParts == 3)
{
if(!headerRead)
{
headerRead = true;
continue;
}
ArrayResize(rulesColumns, n + 1);
ArrayResize(rulesHave, n + 1);
ArrayResize(rulesWant, n + 1);
rulesColumns[n] = (int)StringToInteger(stParts[0]) - 1;
if(rulesColumns[n] < 0)
{
Print("Invalid column number ", rulesColumns[n], " changed to 0");
rulesColumns[n] = 0;
}
rulesHave[n] = stParts[1];
rulesWant[n] = stParts[2];
// Print("Rule ", (n + 1), " for column ", (rulesColumns[n] + 1), ": find '", stParts[1], "', replace with: '", stParts[2], "'");
n++;
}
}
FileClose(h);
return true;
}
public:
static IndexMap *convertReport2Map(const string URL, const bool columnNames = false)
{
if(URL == "")
{
Print("Parameter URL can not be empty");
return NULL;
}
if(RowSelector == "")
{
Print("Enter at least one of parameters: RowSelector, SaveName, TestQuery");
return NULL;
}
HtmlParser p;
string xml;
Print("Reading html-file ", URL);
int h = FileOpen(URL, FILE_READ|FILE_TXT|FILE_SHARE_WRITE|FILE_SHARE_READ|FILE_ANSI, 0, CP_UTF8);
if(h == INVALID_HANDLE)
{
Print("Error reading file '", URL, "': ", GetLastError());
return NULL;
}
StringInit(xml, (int)FileSize(h));
while(!FileIsEnding(h))
{
xml += FileReadString(h) + "\n";
}
// xml = FileReadString(h, (int)FileSize(h)); - has 4095 bytes limit in binary files!
FileClose(h);
int logLevel = 0;
if(LogDomElements) logLevel |= LOG_MASK_DOM;
if(LogDomWarnings) logLevel |= LOG_MASK_DOM_WARNING;
if(LogSelectedElements) logLevel |= LOG_MASK_SELECTOR;
if(logLevel != 0)
{
p.enableDebug(logLevel);
}
DomElement *document = p.parse(xml);
Print("Row selector: '", RowSelector, "'");
string columnSelectors[];
string dataSelectors[];
string headers[];
if(!loadColumnConfig(columnSelectors, dataSelectors, headers)) return NULL;
IndexMap subst;
int rulesColumns[];
string rulesHave[];
string rulesWant[];
if(SubstitutionSettingsFile != "")
{
if(!loadSubstConfig(rulesColumns, rulesHave, rulesWant)) return NULL;
}
for(int i = 0; i < ArraySize(rulesHave); i++)
{
string key = columnSelectors[rulesColumns[i]];
if(!subst.isKeyExisting(key))
{
subst.add(key, new IndexMap("id" + IntegerToString(i)));
}
IndexMap *hm = (IndexMap *)subst[key];
hm.setValue(rulesHave[i], rulesWant[i]);
}
IndexMap *data = document.tableSelect(RowSelector, headers, columnSelectors, dataSelectors, &subst, !columnNames);
return data;
}
};
Reference in New Issue View Git Blame Copy Permalink