Skip to content

Commit

Permalink
#325, #326: Starttimeu and endttimeu implementation (#399)
Browse files Browse the repository at this point in the history
* add EpochTimestamp, TimeQualifier to cleanup TimeStatement.

Missing: tests.

* add tests

* added new variable unquotedValue for epochFromString, changed equals("") check to isEmpty()

* apply spotless
  • Loading branch information
eemhu authored Nov 14, 2024
1 parent 36d4267 commit 12bba5e
Show file tree
Hide file tree
Showing 6 changed files with 621 additions and 132 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -49,21 +49,13 @@
import com.teragrep.pth10.ast.bo.*;
import com.teragrep.pth10.ast.bo.Token.Type;
import com.teragrep.pth10.ast.commands.EmitMode;
import com.teragrep.pth10.ast.time.RelativeTimeParser;
import com.teragrep.pth10.ast.time.RelativeTimestamp;
import com.teragrep.pth_03.antlr.DPLLexer;
import com.teragrep.pth10.ast.time.TimeQualifier;
import com.teragrep.pth_03.antlr.DPLParser;
import com.teragrep.pth_03.antlr.DPLParserBaseVisitor;
import com.teragrep.pth_03.shaded.org.antlr.v4.runtime.tree.TerminalNode;
import org.apache.spark.sql.Column;
import org.apache.spark.sql.functions;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.w3c.dom.Document;
import org.w3c.dom.Element;

import java.sql.Timestamp;
import java.text.ParseException;

/**
* <p>
Expand Down Expand Up @@ -191,65 +183,24 @@ public Node visitTimeQualifier(DPLParser.TimeQualifierContext ctx) {
* @return ElementNode(XML) with LE/GE unixtime
*/
private ElementNode timeQualifierEmitXml(DPLParser.TimeQualifierContext ctx) {
String op = null;
long timevalue = 0;
String value = null;
boolean isRelativeTime = false;
Token comparisonToken;

// Get specifier. We know that 2 first childs are terminals
// 'earliest = '
TerminalNode node = (TerminalNode) ctx.getChild(0);
value = ctx.getChild(1).getText();
Timestamp now = new Timestamp(System.currentTimeMillis());
RelativeTimeParser rtParser = new RelativeTimeParser();
// Is time given as absolute
// Try to check if it is relative and catch exception
try {
// relative time
RelativeTimestamp rtTimestamp = rtParser.parse(value); // might throw NFE if not relative timestamp
timevalue = rtTimestamp.calculate(now);
String value = ctx.getChild(1).getText();

TimeQualifier tq = new TimeQualifier(value, catCtx.getTimeFormatString(), node.getSymbol().getType(), doc);

if (tq.isStartTime()) {
startTime = tq.epoch();
}
catch (NumberFormatException ne) {
// absolute time
timevalue = this.getEpochFromString(value, catCtx.getTimeFormatString());
else if (tq.isEndTime()) {
endTime = tq.epoch();
}
// Handle date calculations
switch (node.getSymbol().getType()) {
case DPLLexer.EARLIEST: {
op = "earliest";
comparisonToken = new Token(Type.GE);

startTime = timevalue;
break;
}
case DPLLexer.INDEX_EARLIEST: {
op = "index_earliest";
comparisonToken = new Token(Type.GE);
startTime = timevalue;
break;
}
case DPLLexer.LATEST: {
op = "latest";
comparisonToken = new Token(Type.LE);
endTime = timevalue;
break;
}
case DPLLexer.INDEX_LATEST: {
op = "index_latest";
comparisonToken = new Token(Type.LE);
endTime = timevalue;
break;
}
default: {
throw new RuntimeException("TimeQualifier <" + node.getSymbol().getText() + "> not implemented yet.");
}
else {
throw new UnsupportedOperationException("Unexpected token: " + node.getSymbol().getText());
}

Element el = doc.createElement(op);
el.setAttribute("operation", comparisonToken.toString());
el.setAttribute("value", Long.toString(timevalue));
return new ElementNode(el);
return new ElementNode(tq.xmlElement());
}

/**
Expand All @@ -261,78 +212,21 @@ private ElementNode timeQualifierEmitXml(DPLParser.TimeQualifierContext ctx) {
* @return ColumnNode with leq/geq unixtime
*/
private ColumnNode timeQualifierEmitCatalyst(DPLParser.TimeQualifierContext ctx) {
String op = null;
Column rv = null;
long timevalue = 0;
String value = null;
boolean isRelativeTime = false;
Token comparisonToken;

// Get specifier. We know that 2 first childs are terminals
// 'earliest = '
TerminalNode node = (TerminalNode) ctx.getChild(0);
value = ctx.getChild(1).getText();
Timestamp now = new Timestamp(System.currentTimeMillis());
RelativeTimeParser rtParser = new RelativeTimeParser();
// Is time given as absolute
// Try to check if it is relative and catch exception
try {
// relative time
RelativeTimestamp rtTimestamp = rtParser.parse(value);
timevalue = rtTimestamp.calculate(now);
}
catch (NumberFormatException ne) {
// absolute time
timevalue = this.getEpochFromString(value, catCtx.getTimeFormatString());
}
String value = ctx.getChild(1).getText();

Column col = new Column("`_time`");
// Handle date calculations
switch (node.getSymbol().getType()) {
case DPLLexer.EARLIEST:
case DPLLexer.INDEX_EARLIEST: {
startTime = timevalue;
TimeQualifier tq = new TimeQualifier(value, catCtx.getTimeFormatString(), node.getSymbol().getType(), doc);

//java.sql.Timestamp tt = new Timestamp(timevalue*1000);
//rv = col.geq(tt);
rv = col.geq(functions.from_unixtime(functions.lit(timevalue)));
break;
}
case DPLLexer.LATEST:
case DPLLexer.INDEX_LATEST: {
endTime = timevalue;
rv = col.lt(functions.from_unixtime(functions.lit(timevalue)));
break;
}
default: {
throw new RuntimeException("TimeQualifier <" + node.getSymbol().getText() + "> not implemented yet.");
}
if (tq.isStartTime()) {
startTime = tq.epoch();
}

return new ColumnNode(rv);
}

// Uses defaultTimeFormat if timeformat is null and DPLTimeFormat if timeformat isn't null (which means that the
// timeformat= option was used).
private long getEpochFromString(String value, String timeFormatString) {
value = new UnquotedText(new TextString(value)).read(); // erase the possible outer quotes
long timevalue = 0;
if (timeFormatString == null || timeFormatString.equals("")) {
timevalue = new DefaultTimeFormat().getEpoch(value);
else if (tq.isEndTime()) {
endTime = tq.epoch();
}
else {
// TODO: should be included in DPLTimeFormat
if (timeFormatString.equals("%s")) {
return Long.parseLong(value);
}
try {
timevalue = new DPLTimeFormat(timeFormatString).getEpoch(value);
}
catch (ParseException e) {
throw new RuntimeException("TimeQualifier conversion error: <" + value + "> can't be parsed.");
}
throw new UnsupportedOperationException("Unexpected token: " + node.getSymbol().getText());
}
return timevalue;
}

return new ColumnNode(tq.column());
}
}
117 changes: 117 additions & 0 deletions src/main/java/com/teragrep/pth10/ast/time/EpochTimestamp.java
Original file line number Diff line number Diff line change
@@ -0,0 +1,117 @@
/*
* Teragrep Data Processing Language (DPL) translator for Apache Spark (pth_10)
* Copyright (C) 2019-2024 Suomen Kanuuna Oy
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*
*
* Additional permission under GNU Affero General Public License version 3
* section 7
*
* If you modify this Program, or any covered work, by linking or combining it
* with other code, such other code is not for that reason alone subject to any
* of the requirements of the GNU Affero GPL version 3 as long as this Program
* is the same Program as licensed from Suomen Kanuuna Oy without any additional
* modifications.
*
* Supplemented terms under GNU Affero General Public License version 3
* section 7
*
* Origin of the software must be attributed to Suomen Kanuuna Oy. Any modified
* versions must be marked as "Modified version of" The Program.
*
* Names of the licensors and authors may not be used for publicity purposes.
*
* No rights are granted for use of trade names, trademarks, or service marks
* which are in The Program if any.
*
* Licensee must indemnify licensors and authors for any liability that these
* contractual assumptions impose on licensors and authors.
*
* To the extent this program is licensed as part of the Commercial versions of
* Teragrep, the applicable Commercial License may apply to this file if you as
* a licensee so wish it.
*/
package com.teragrep.pth10.ast.time;

import com.teragrep.pth10.ast.DPLTimeFormat;
import com.teragrep.pth10.ast.DefaultTimeFormat;
import com.teragrep.pth10.ast.TextString;
import com.teragrep.pth10.ast.UnquotedText;

import java.sql.Timestamp;
import java.text.ParseException;
import java.util.Objects;

public final class EpochTimestamp {

private final String value;
private final String timeformat;

public EpochTimestamp(final String value, final String timeformat) {
this.value = value;
this.timeformat = timeformat;
}

public long epoch() {
long rv;
try {
RelativeTimestamp relativeTimestamp = new RelativeTimeParser().parse(value);
rv = relativeTimestamp.calculate(new Timestamp(System.currentTimeMillis()));
}
catch (NumberFormatException ne) {
rv = epochFromString(value, timeformat);
}

return rv;
}

// Uses defaultTimeFormat if timeformat is null and DPLTimeFormat if timeformat isn't null (which means that the
// timeformat= option was used).
private long epochFromString(final String value, final String timeFormatString) {
final String unquotedValue = new UnquotedText(new TextString(value)).read(); // erase the possible outer quotes
final long timevalue;
if (timeFormatString == null || timeFormatString.isEmpty()) {
timevalue = new DefaultTimeFormat().getEpoch(unquotedValue);
}
else {
// TODO: should be included in DPLTimeFormat
if (timeFormatString.equals("%s")) {
return Long.parseLong(unquotedValue);
}
try {
timevalue = new DPLTimeFormat(timeFormatString).getEpoch(unquotedValue);
}
catch (ParseException e) {
throw new RuntimeException("TimeQualifier conversion error: <" + unquotedValue + "> can't be parsed.");
}
}
return timevalue;
}

@Override
public boolean equals(Object o) {
if (this == o)
return true;
if (o == null || getClass() != o.getClass())
return false;
EpochTimestamp that = (EpochTimestamp) o;
return Objects.equals(value, that.value) && Objects.equals(timeformat, that.timeformat);
}

@Override
public int hashCode() {
return Objects.hash(value, timeformat);
}
}
Loading

0 comments on commit 12bba5e

Please sign in to comment.