Commit a202b93f authored by Michał Komar's avatar Michał Komar
Browse files

refs #1023: Pobieranie przykładowych danych z tabeli użytkownika

parent 09694547
......@@ -17,7 +17,6 @@ import java.util.stream.StreamSupport;
import org.apache.commons.collections4.KeyValue;
import org.apache.commons.collections4.keyvalue.DefaultKeyValue;
import org.apache.commons.lang.StringUtils;
import org.apache.commons.lang3.math.NumberUtils;
import org.apache.commons.lang3.reflect.FieldUtils;
import org.apache.commons.lang3.reflect.MethodUtils;
import org.springframework.batch.item.ItemProcessor;
......
......@@ -6,6 +6,7 @@ import org.springframework.security.access.annotation.Secured;
import pl.edu.icm.cocos.services.api.model.metadata.CocosMetadata;
import pl.edu.icm.cocos.services.api.model.metadata.CocosUserTable;
import pl.edu.icm.cocos.services.api.model.query.CocosQueryExecution;
public interface CocosDatabaseMetadataService {
......@@ -22,4 +23,6 @@ public interface CocosDatabaseMetadataService {
String getUserDatabasePlaceholder();
CocosQueryExecution getSampleData(String tableName);
}
......@@ -26,6 +26,7 @@ import com.cloudera.beeswax.api.Results;
import com.cloudera.beeswax.api.ResultsMetadata;
import pl.edu.icm.cocos.services.api.exceptions.CocosQueryErrorException;
import pl.edu.icm.cocos.services.query.executor.CocosQueryMetadataMapper;
public class ImpalaDatabaseClient {
......@@ -41,6 +42,9 @@ public class ImpalaDatabaseClient {
@Autowired
private ImpalaClientHolder clientHolder;
@Autowired
protected CocosQueryMetadataMapper metadataMapper;
private Client getClient() {
try {
......@@ -61,20 +65,22 @@ public class ImpalaDatabaseClient {
}
@Transactional(transactionManager = "impalaTransactionManager", propagation = Propagation.SUPPORTS)
public List<List<String>> executeImmediately(String userQuery) {
public ImpalaExecutionResult executeImmediately(String userQuery) {
Query query = new Query(userQuery, null, hadoopUser);
Client client = getClient();
List<List<String>> results;
QueryHandle handle = null;
try {
ImpalaExecutionResult result = new ImpalaExecutionResult();
handle = client.query(query);
results = getData(handle);
ResultsMetadata metadata = getMetadata(handle);
result.setColumns(metadataMapper.mapMetadata(metadata).getValues());
result.setRows(getData(handle));
return result;
} catch (TException e) {
throw new CocosQueryErrorException("Couldn't execute query", e);
} finally {
closeQueryQuietly(handle);
}
return results;
}
public QueryExplanation executeExplain(String userQuery) {
......
package pl.edu.icm.cocos.services.database.impala;
import java.io.Serializable;
import java.util.ArrayList;
import java.util.List;
public class ImpalaExecutionResult implements Serializable {
private static final long serialVersionUID = 1343412638349090028L;
private List<List<String>> rows = new ArrayList<>();
private List<String> columns = new ArrayList<>();
public List<List<String>> getRows() {
return rows;
}
public void setRows(List<List<String>> rows) {
this.rows = rows;
}
public List<String> getColumns() {
return columns;
}
public void setColumns(List<String> columns) {
this.columns = columns;
}
}
package pl.edu.icm.cocos.services.metadata;
import java.util.List;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.stream.Collectors;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
......@@ -23,9 +25,12 @@ import pl.edu.icm.cocos.services.api.model.metadata.CocosTable;
import pl.edu.icm.cocos.services.api.model.metadata.CocosUserTable;
import pl.edu.icm.cocos.services.api.model.query.CocosQueryExecution;
import pl.edu.icm.cocos.services.api.model.query.CocosQueryExecutionStatus;
import pl.edu.icm.cocos.services.api.model.query.CocosQueryResultMetadata;
import pl.edu.icm.cocos.services.api.model.query.CocosQueryResultRow;
import pl.edu.icm.cocos.services.api.model.query.CocosUserCreateTableQuery;
import pl.edu.icm.cocos.services.api.model.query.events.CocosSimulationQueryQueueChangeEvent;
import pl.edu.icm.cocos.services.database.impala.ImpalaDatabaseClient;
import pl.edu.icm.cocos.services.database.impala.ImpalaExecutionResult;
import pl.edu.icm.cocos.services.database.repositories.CocosFunctionRepository;
import pl.edu.icm.cocos.services.database.repositories.CocosTableRepository;
import pl.edu.icm.cocos.services.database.repositories.CocosUserTableRepository;
......@@ -39,6 +44,9 @@ public class CocosDatabaseMetadataServiceImpl implements CocosDatabaseMetadataSe
@Value("${cocos.impala.databasePrefix}")
private String databasePrefix;
@Value("${cocos.impala.tableSampleDataLimit}")
private String sampleDataLimit;
@Autowired
private CocosTableRepository tableRepository;
......@@ -59,7 +67,7 @@ public class CocosDatabaseMetadataServiceImpl implements CocosDatabaseMetadataSe
@Override
@Transactional(readOnly = true)
@Cacheable(value="getMetadata")
@Cacheable(value = "getMetadata")
public CocosMetadata getMetadata() {
List<CocosTable> tables = tableRepository.findAll();
List<CocosFunction> functions = functionRepository.findAll();
......@@ -81,8 +89,8 @@ public class CocosDatabaseMetadataServiceImpl implements CocosDatabaseMetadataSe
userTable.setDescription(createTableQuery.getDescription());
userTableRepository.save(userTable);
String tableName = CocosDatabaseMetadataUtils.getTablename(userTable);
List<List<String>> results = databaseClient.executeImmediately("DESCRIBE " + databasePrefix + tableName + ";");
for (List<String> data : results) {
ImpalaExecutionResult result = databaseClient.executeImmediately("DESCRIBE " + databasePrefix + tableName + ";");
for (List<String> data : result.getRows()) {
CocosColumn cocosColumn = new CocosColumn();
cocosColumn.setTable(userTable);
cocosColumn.setName(data.get(0));
......@@ -114,6 +122,30 @@ public class CocosDatabaseMetadataServiceImpl implements CocosDatabaseMetadataSe
return userDatabasePlaceholder;
}
@Override
@Transactional(transactionManager = "impalaTransactionManager")
public CocosQueryExecution getSampleData(String tableName) {
CocosUserTable table = getUserTableByName(tableName);
String queryTableName = CocosDatabaseMetadataUtils.getTablename(table);
ImpalaExecutionResult result = databaseClient
.executeImmediately("SELECT * FROM " + databasePrefix + queryTableName + " LIMIT " + sampleDataLimit + ";");
final CocosQueryExecution execution = new CocosQueryExecution();
CocosQueryResultMetadata metadata = new CocosQueryResultMetadata();
metadata.setValues(result.getColumns());
metadata.setQueryResult(execution);
execution.setMetadata(metadata);
AtomicInteger rowNumber = new AtomicInteger();
List<CocosQueryResultRow> rows = result.getRows().stream().sequential().map(r -> {
CocosQueryResultRow row = new CocosQueryResultRow();
row.setQueryResult(execution);
row.setRowNumber(rowNumber.getAndIncrement());
row.setValues(r);
return row;
}).collect(Collectors.toList());
execution.setRows(rows);
return execution;
}
@Override
@Transactional
public void removeUserTable(String userTable) {
......
......@@ -4,7 +4,6 @@ import java.util.List;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.stream.Collectors;
import org.apache.hadoop.hive.metastore.api.FieldSchema;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Component;
......@@ -22,6 +21,9 @@ public class CocosImpalaResultsMapper {
@Autowired
private CocosQueryRowMapper rowMapper;
@Autowired
private CocosQueryMetadataMapper metadataMapper;
@Value("${cocos.impala.maxColumns}")
private Integer maxColumns;
......@@ -36,11 +38,7 @@ public class CocosImpalaResultsMapper {
return row;
}).collect(Collectors.toList());
CocosQueryResultMetadata metadata = new CocosQueryResultMetadata();
for (FieldSchema column : resultsMetadata.getSchema().getFieldSchemas()) {
metadata.getValues().add(column.getName() + " (" + column.getType() + ")");
}
CocosQueryResultMetadata metadata = metadataMapper.mapMetadata(resultsMetadata);
reduceColumns(metadata);
queryResult.setStatus(CocosQueryExecutionStatus.SUCCESS);
queryResult.setRows(rows);
......
package pl.edu.icm.cocos.services.query.executor;
import org.apache.hadoop.hive.metastore.api.FieldSchema;
import org.springframework.stereotype.Component;
import com.cloudera.beeswax.api.ResultsMetadata;
import pl.edu.icm.cocos.services.api.model.query.CocosQueryResultMetadata;
@Component
public class CocosQueryMetadataMapper {
public CocosQueryResultMetadata mapMetadata(ResultsMetadata resultsMetadata) {
CocosQueryResultMetadata metadata = new CocosQueryResultMetadata();
for (FieldSchema column : resultsMetadata.getSchema().getFieldSchemas()) {
metadata.getValues().add(column.getName() + " (" + column.getType() + ")");
}
return metadata;
}
}
......@@ -15,6 +15,7 @@ cocos.impala.user=cocos
cocos.impala.fetchSize=1000
cocos.impala.maxColumns=100
cocos.impala.databasePrefix=
cocos.impala.tableSampleDataLimit=20
cocos.query.maxAnonymous=2
cocos.query.user.databasePlaceholder=MY_TABLES
......
......@@ -170,6 +170,10 @@ public class QueryServiceTest extends AbstractTestNGSpringContextTests {
Assertions.assertThat(userTable.getColumns()).isNotNull();
Assertions.assertThat(query.getUserTable()).isNotNull();
Assertions.assertThat(query.getUserTable()).isEqualByComparingTo(userTable);
CocosQueryExecution execution = metadataService.getSampleData("test_table");
Assertions.assertThat(execution).isNotNull();
Assertions.assertThat(execution.getRows()).hasSize(10);
}
private void dropImpalaTableIfExists(String tableName) {
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment