Skip to content

Enable xml type support for oracle for dts-connectors only #657

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
vanshikaagupta22 wants to merge 1 commit into
base: develop
Choose a base branch
from
Open

Enable xml type support for oracle for dts-connectors only #657

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleConnector.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -115,7 +115,8 @@ protected DBConnectorPath getDBConnectorPath(String path) {
protected SchemaReader getSchemaReader(String sessionID) {
return new OracleSourceSchemaReader(sessionID, config.getTreatAsOldTimestamp(),
config.getTreatPrecisionlessNumAsDeci(),
config.getTreatTimestampLTZAsTimestamp());
config.getTreatTimestampLTZAsTimestamp(),
config.getXmlTypeEnabled());
}

@Override
Expand Down
16 changes: 14 additions & 2 deletions oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleConnectorConfig.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -42,14 +42,15 @@ public OracleConnectorConfig(String host, int port, String user, String password
public OracleConnectorConfig(String host, int port, String user, String password, String jdbcPluginName,
String connectionArguments, String connectionType, String database) {
this(host, port, user, password, jdbcPluginName, connectionArguments, connectionType, database, null, null, null,
null, null);
null, null, null);
}

public OracleConnectorConfig(String host, int port, String user, String password, String jdbcPluginName,
String connectionArguments, String connectionType, String database,
String role, Boolean useSSL, @Nullable Boolean treatAsOldTimestamp,
@Nullable Boolean treatPrecisionlessNumAsDeci,
@Nullable Boolean treatTimestampLTZAsTimestamp) {
@Nullable Boolean treatTimestampLTZAsTimestamp,
@Nullable Boolean enableXmlType) {

this.host = host;
this.port = port;
Expand All @@ -64,6 +65,7 @@ public OracleConnectorConfig(String host, int port, String user, String password
this.treatAsOldTimestamp = treatAsOldTimestamp;
this.treatPrecisionlessNumAsDeci = treatPrecisionlessNumAsDeci;
this.treatTimestampLTZAsTimestamp = treatTimestampLTZAsTimestamp;
this.enableXmlType = enableXmlType;
}

@Override
Expand Down Expand Up @@ -105,6 +107,12 @@ public String getConnectionString() {
@Nullable
public Boolean treatTimestampLTZAsTimestamp;

@Name(OracleConstants.ENABLE_XML_TYPE)
@Description("A hidden field to handle mapping of Oracle XML type to BQ String.")
@Nullable
public Boolean enableXmlType;


@Override
protected int getDefaultPort() {
return 1521;
Expand Down Expand Up @@ -139,6 +147,10 @@ public Boolean getTreatTimestampLTZAsTimestamp() {
return Boolean.TRUE.equals(treatTimestampLTZAsTimestamp);
}

public Boolean getXmlTypeEnabled() {
return Boolean.TRUE.equals(enableXmlType);
}

@Override
public Properties getConnectionArgumentsProperties() {
Properties prop = super.getConnectionArgumentsProperties();
Expand Down
1 change: 1 addition & 0 deletions oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleConstants.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,7 @@ private OracleConstants() {
public static final String TREAT_AS_OLD_TIMESTAMP = "treatAsOldTimestamp";
public static final String TREAT_PRECISIONLESSNUM_AS_DECI = "treatPrecisionlessNumAsDeci";
public static final String TREAT_TIMESTAMP_LTZ_AS_TIMESTAMP = "treatTimestampLTZAsTimestamp";
public static final String ENABLE_XML_TYPE = "enableXmlType";

/**
* Constructs the Oracle connection string based on the provided connection type, host, port, and database.
Expand Down
9 changes: 6 additions & 3 deletions oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleSource.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -69,9 +69,10 @@ protected SchemaReader getSchemaReader() {
boolean treatAsOldTimestamp = oracleSourceConfig.getConnection().getTreatAsOldTimestamp();
boolean treatPrecisionlessNumAsDeci = oracleSourceConfig.getConnection().getTreatPrecisionlessNumAsDeci();
boolean treatTimestampLTZAsTimestamp = oracleSourceConfig.getConnection().getTreatTimestampLTZAsTimestamp();
boolean enableXmlType = oracleSourceConfig.getConnection().getXmlTypeEnabled();

return new OracleSourceSchemaReader(null, treatAsOldTimestamp, treatPrecisionlessNumAsDeci,
treatTimestampLTZAsTimestamp);
treatTimestampLTZAsTimestamp, enableXmlType);
}

@Override
Expand Down Expand Up @@ -139,10 +140,12 @@ public OracleSourceConfig(String host, int port, String user, String password, S
int defaultBatchValue, int defaultRowPrefetch,
String importQuery, Integer numSplits, int fetchSize,
String boundingQuery, String splitBy, Boolean useSSL, Boolean treatAsOldTimestamp,
Boolean treatPrecisionlessNumAsDeci, Boolean treatTimestampLTZAsTimestamp) {
Boolean treatPrecisionlessNumAsDeci, Boolean treatTimestampLTZAsTimestamp,
Boolean enableXmlType) {
this.connection = new OracleConnectorConfig(host, port, user, password, jdbcPluginName, connectionArguments,
connectionType, database, role, useSSL, treatAsOldTimestamp,
treatPrecisionlessNumAsDeci, treatTimestampLTZAsTimestamp);
treatPrecisionlessNumAsDeci, treatTimestampLTZAsTimestamp,
enableXmlType);
this.defaultBatchValue = defaultBatchValue;
this.defaultRowPrefetch = defaultRowPrefetch;
this.fetchSize = fetchSize;
Expand Down
10 changes: 8 additions & 2 deletions oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleSourceSchemaReader.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -61,6 +61,7 @@ public class OracleSourceSchemaReader extends CommonSchemaReader {
BINARY_DOUBLE,
BFILE,
LONG,
Types.SQLXML,
LONG_RAW,
Types.NUMERIC,
Types.DECIMAL
Expand All @@ -70,16 +71,19 @@ public class OracleSourceSchemaReader extends CommonSchemaReader {
private final Boolean isTimestampOldBehavior;
private final Boolean isPrecisionlessNumAsDecimal;
private final Boolean isTimestampLtzFieldTimestamp;
private final Boolean isXmlTypeEnabled;

public OracleSourceSchemaReader() {
this(null, false, false, false);
this(null, false, false, false, false);
}
public OracleSourceSchemaReader(@Nullable String sessionID, boolean isTimestampOldBehavior,
boolean isPrecisionlessNumAsDecimal, boolean isTimestampLtzFieldTimestamp) {
boolean isPrecisionlessNumAsDecimal, boolean isTimestampLtzFieldTimestamp,
boolean isXmlTypeEnabled) {
this.sessionID = sessionID;
this.isTimestampOldBehavior = isTimestampOldBehavior;
this.isPrecisionlessNumAsDecimal = isPrecisionlessNumAsDecimal;
this.isTimestampLtzFieldTimestamp = isTimestampLtzFieldTimestamp;
this.isXmlTypeEnabled = isXmlTypeEnabled;
}

@Override
Expand All @@ -104,6 +108,8 @@ public Schema getSchema(ResultSetMetaData metadata, int index) throws SQLExcepti
case INTERVAL_YM:
case LONG:
return Schema.of(Schema.Type.STRING);
case Types.SQLXML:
Copy link
Copy Markdown
Contributor

@itsankit-google itsankit-google Apr 24, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

we should add a comment explaining for future context that it is only added to support DTS connector and currently not in a working state in CDAP plugin.

return isXmlTypeEnabled ? Schema.of(Schema.Type.STRING) : super.getSchema(metadata, index);
case Types.NUMERIC:
case Types.DECIMAL:
// FLOAT and REAL are returned as java.sql.Types.NUMERIC but with value that is a java.lang.Double
Expand Down
39 changes: 37 additions & 2 deletions oracle-plugin/src/test/java/io/cdap/plugin/oracle/OracleSchemaReaderTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@

import com.google.common.collect.Lists;
import io.cdap.cdap.api.data.schema.Schema;
import io.cdap.cdap.api.exception.ProgramFailureException;
import org.junit.Assert;
import org.junit.Test;
import org.junit.runner.RunWith;
Expand All @@ -27,13 +28,14 @@
import java.sql.ResultSet;
import java.sql.ResultSetMetaData;
import java.sql.SQLException;
import java.sql.Types;
import java.util.List;

public class OracleSchemaReaderTest {

@Test
public void getSchema_timestampLTZFieldTrue_returnTimestamp() throws SQLException {
OracleSourceSchemaReader schemaReader = new OracleSourceSchemaReader(null, false, false, true);
OracleSourceSchemaReader schemaReader = new OracleSourceSchemaReader(null, false, false, true, false);

ResultSet resultSet = Mockito.mock(ResultSet.class);
ResultSetMetaData metadata = Mockito.mock(ResultSetMetaData.class);
Expand Down Expand Up @@ -64,7 +66,7 @@ public void getSchema_timestampLTZFieldTrue_returnTimestamp() throws SQLExceptio

@Test
public void getSchema_timestampLTZFieldFalse_returnDatetime() throws SQLException {
OracleSourceSchemaReader schemaReader = new OracleSourceSchemaReader(null, false, false, false);
OracleSourceSchemaReader schemaReader = new OracleSourceSchemaReader(null, false, false, false, false);

ResultSet resultSet = Mockito.mock(ResultSet.class);
ResultSetMetaData metadata = Mockito.mock(ResultSetMetaData.class);
Expand All @@ -91,4 +93,37 @@ public void getSchema_timestampLTZFieldFalse_returnDatetime() throws SQLExceptio
Assert.assertEquals(expectedSchemaFields.get(1).getName(), actualSchemaFields.get(1).getName());
Assert.assertEquals(expectedSchemaFields.get(1).getSchema(), actualSchemaFields.get(1).getSchema());
}

@Test
public void getSchema_xmlField_returnString() throws SQLException {
Comment thread
vanshikaagupta22 marked this conversation as resolved.
OracleSourceSchemaReader schemaReader = new OracleSourceSchemaReader(null, false, false, false, true);
ResultSet resultSet = Mockito.mock(ResultSet.class);
ResultSetMetaData metadata = Mockito.mock(ResultSetMetaData.class);
Mockito.when(resultSet.getMetaData()).thenReturn(metadata);
Mockito.when(metadata.getColumnCount()).thenReturn(1);
Mockito.when(metadata.getColumnType(1)).thenReturn(Types.SQLXML);
Mockito.when(metadata.getColumnName(1)).thenReturn("xmlData");

List<Schema.Field> actualSchemaFields = schemaReader.getSchemaFields(resultSet);

List<Schema.Field> expectedSchemaFields = Lists.newArrayList();
expectedSchemaFields.add(Schema.Field.of("xmlData", Schema.of(Schema.Type.STRING)));
Assert.assertEquals(expectedSchemaFields.get(0).getName(), actualSchemaFields.get(0).getName());
Assert.assertEquals(expectedSchemaFields.get(0).getSchema(), actualSchemaFields.get(0).getSchema());
}

@Test
public void getSchema_xmlFieldDisabled_throwsProgramFailureException() throws SQLException {
OracleSourceSchemaReader schemaReader = new OracleSourceSchemaReader(null,
false, false, false, false);
ResultSet resultSet = Mockito.mock(ResultSet.class);
ResultSetMetaData metadata = Mockito.mock(ResultSetMetaData.class);
Mockito.when(resultSet.getMetaData()).thenReturn(metadata);
Mockito.when(metadata.getColumnCount()).thenReturn(1);
Mockito.when(metadata.getColumnType(1)).thenReturn(Types.SQLXML);
Mockito.when(metadata.getColumnName(1)).thenReturn("xmlData");

Assert.assertThrows(ProgramFailureException.class, () -> schemaReader.getSchemaFields(resultSet));

}
}
19 changes: 19 additions & 0 deletions oracle-plugin/widgets/Oracle-batchsource.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -177,6 +177,25 @@
]
}
},
{
"widget-type": "hidden",
"label": "Enable Xml Type",
"name": "enableXmlType",
"widget-attributes": {
"layout": "inline",
"default": "false",
"options": [
{
"id": "true",
"label": "true"
},
{
"id": "false",
"label": "false"
}
]
}
},
{
"name": "connectionType",
"label": "Connection Type",
Expand Down
19 changes: 19 additions & 0 deletions oracle-plugin/widgets/Oracle-connector.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -186,6 +186,25 @@
}
]
}
},
{
"widget-type": "hidden",
"label": "Enable Xml Type",
"name": "enableXmlType",
"widget-attributes": {
"layout": "inline",
"default": "false",
"options": [
{
"id": "true",
"label": "true"
},
{
"id": "false",
"label": "false"
}
]
}
}
]
},
Expand Down
Loading