<?php
header("Content-Type: text/plain; charset=UTF-8");
require_once "classes/parser/tokenizer.php";
// splits an string into 'words'
// Note that you DO NOT NEED to write an explicit regular expression. Instead of typing "/^\s*\w+/" we can write "\w+".
// In this case, the function ignores the left spaces and start searching from the current offset position.
$t = new Tokenizer("Lorem ipsum dolor sit amet");
while (list($token) = $t->match("\w+")) {
echo "$token-";
}
echo "\n";
// In any case, you can still use explicit regular expressions:
$t = new Tokenizer("I'm 35 years old");
if (list($years) = $t->match("/\d+/")) {
echo "You are $years old\n";
}
// flags example.
// OFFSET_CAPTURE: retrieves offset positions
// CASE_SENSITIVE: switches to 'case sensitive' mode
// SEARCH_ANYWHERE: start searching from any position
$t = new Tokenizer("select * From table where id = 100", Tokenizer::OFFSET_CAPTURE|Tokenizer::CASE_SENSITIVE|Tokenizer::SEARCH_ANYWHERE);
if ($t->match("From\s+(\w+)", $matches)) {
print_r($matches);
}
// parses a basic SQL sentence
$t = new Tokenizer("Select Id, Name, Age From users Where Id = 101");
if ($t->match("select")) {
// columns
$columns = array();
while (list($column) = $t->match("\w+")) {
array_push($columns, $column);
if (!$t->match(",")) {
break;
}
}
// 'from' clause
if ($t->match("from\s+(\w+)", $matches)) {
$table_name = $matches[1];
echo "You want to get the columns " . implode(", ", $columns) . " from the table $table_name.\n";
}
}
// splits a phrase into tokens
$t = new Tokenizer("Look at my horse, my horse is amazing");
while (list($token) = $t->token()) {
echo "$token-";
}
echo "\n";
// is the next token equal to a given string?
$t = new Tokenizer("Lorem Ipsum Dolor");
if (list($str) = $t->eq("lorem ipsum dolor")) {
echo "$str Sit Amet\n";
}
// is the next token a number?
$t = new Tokenizer("125.48E-4");
if (list($number) = $t->number()) {
echo "Yes, it is a float number: $number\n";
}
// is the next token a string?
$t = new Tokenizer(' "Houston, we have a \"problem\"." ');
if (list($str) = $t->str()) {
echo "Yes, is is a string: $str\n";
}
// and the next, is a string?
$t = new Tokenizer(" 'Look at my horse, my \'horse\' is amazing' ");
if (list($str) = $t->str()) {
echo "Yes, is is a string too: $str\n";
}
// is the next token an identifier?
$t = new Tokenizer("Id: hello093");
if ($t->eq("Id:") && list($id) = $t->identifier()) {
echo "Yes, it is an identifier: $id\n";
}
|
Download
