diff --git a/signalData/resources/web/signaldata/QCView/DataService.js b/signalData/resources/web/signaldata/QCView/DataService.js index 2c68d5ca2..d96a86d83 100644 --- a/signalData/resources/web/signaldata/QCView/DataService.js +++ b/signalData/resources/web/signaldata/QCView/DataService.js @@ -184,10 +184,11 @@ Ext4.define('LABKEY.SignalData.DataService', { var context = { RunIds: runIds, - DataNames: dataNames + DataNames: dataNames, + ProtocolName: schema.lastIndexOf('.') > 0 ? schema.substring(schema.lastIndexOf('.') + 1, schema.length) : '' }, _count = 0; - var loader = function() { + var loader = function() { _count++; if (_count == 4) { @@ -301,14 +302,14 @@ Ext4.define('LABKEY.SignalData.DataService', { // // Get the associated Assay information // - this.getAssayDefinition('Signal Data', function(def) { + this.getAssayDefinition('Signal Data', context.ProtocolName, function(def) { context.AssayDefinition = def; loader(); }, this); // // Get the associated HPLC Assay information // - this.getAssayDefinition('HPLC', function(def) { + this.getAssayDefinition('HPLC', context.ProtocolName, function(def) { context.HPLCDefinition = def; loader(); }, this); @@ -340,21 +341,28 @@ Ext4.define('LABKEY.SignalData.DataService', { } }, - getAssayDefinition : function(assayType /* String */, callback, scope) { - //TODO: can we do this by ID? + createAssayDefinitionKey : function(assayType, protocolName) { + return assayType + ':' + protocolName; + }, + + getAssayDefinition : function(assayType /* String */, protocolName, callback, scope) { if (Ext4.isString(assayType)) { - if (Ext4.isObject(this._AssayTypeCache[assayType])) { + const cacheKey = this.createAssayDefinitionKey(assayType, protocolName); + if (Ext4.isObject(this._AssayTypeCache[cacheKey])) { if (Ext4.isFunction(callback)) { - callback.call(scope || this, this._AssayTypeCache[assayType]); + callback.call(scope || this, this._AssayTypeCache[cacheKey]); } } else { LABKEY.Assay.getByType({ type: assayType, success: function(defs) { - this._AssayTypeCache[assayType] = defs[0]; + Ext4.each(defs, function(def) { + this._AssayTypeCache[this.createAssayDefinitionKey(assayType, def.name)] = def; + }, this); + if (Ext4.isFunction(callback)) { - callback.call(scope || this, this._AssayTypeCache[assayType]); + callback.call(scope || this, this._AssayTypeCache[cacheKey]); } }, scope: this diff --git a/signalData/src/org/labkey/signaldata/SignalDataModule.java b/signalData/src/org/labkey/signaldata/SignalDataModule.java index fd4dc018d..1ec2dcdea 100644 --- a/signalData/src/org/labkey/signaldata/SignalDataModule.java +++ b/signalData/src/org/labkey/signaldata/SignalDataModule.java @@ -19,16 +19,16 @@ import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; import org.labkey.api.data.UpgradeCode; -import org.labkey.api.module.DefaultModule; import org.labkey.api.module.ModuleContext; import org.labkey.api.module.ModuleProperty; +import org.labkey.api.module.SpringModule; import org.labkey.api.view.WebPartFactory; import java.util.Collection; import java.util.Collections; import java.util.List; -public class SignalDataModule extends DefaultModule +public class SignalDataModule extends SpringModule { public static final String NAME = "SignalData"; public static final String QC_PROVIDER_PROPERTY_NAME = "QCViewProviderModule"; @@ -64,11 +64,6 @@ protected void init() addController(SignalDataController.NAME, SignalDataController.class); } - @Override - public void doStartup(ModuleContext moduleContext) - { - } - @Override public @Nullable Double getSchemaVersion() { @@ -93,4 +88,8 @@ public boolean hasScripts() return new SignalDataUpgradeCode(); } + @Override + protected void startupAfterSpringConfig(ModuleContext moduleContext) + { + } } \ No newline at end of file diff --git a/signalData/src/org/labkey/signaldata/pipeline/SignalDataImportTask.java b/signalData/src/org/labkey/signaldata/pipeline/SignalDataImportTask.java new file mode 100644 index 000000000..bf6101d9f --- /dev/null +++ b/signalData/src/org/labkey/signaldata/pipeline/SignalDataImportTask.java @@ -0,0 +1,392 @@ +package org.labkey.signaldata.pipeline; + +import org.apache.commons.lang3.StringUtils; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.logging.log4j.Logger; +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; +import org.labkey.api.assay.AssayProvider; +import org.labkey.api.assay.AssayRunUploadContext; +import org.labkey.api.assay.AssayService; +import org.labkey.api.assay.DefaultAssayRunCreator; +import org.labkey.api.assay.transform.DataTransformService; +import org.labkey.api.collections.CaseInsensitiveHashMap; +import org.labkey.api.data.Container; +import org.labkey.api.data.ContainerManager; +import org.labkey.api.dataiterator.MapDataIterator; +import org.labkey.api.exp.ExperimentException; +import org.labkey.api.exp.api.ExpData; +import org.labkey.api.exp.api.ExpProtocol; +import org.labkey.api.exp.api.ExperimentService; +import org.labkey.api.exp.query.ExpDataTable; +import org.labkey.api.files.FileContentService; +import org.labkey.api.pipeline.AbstractTaskFactory; +import org.labkey.api.pipeline.AbstractTaskFactorySettings; +import org.labkey.api.pipeline.PipeRoot; +import org.labkey.api.pipeline.PipelineJob; +import org.labkey.api.pipeline.PipelineService; +import org.labkey.api.pipeline.RecordedActionSet; +import org.labkey.api.pipeline.file.FileAnalysisJobSupport; +import org.labkey.api.query.ValidationException; +import org.labkey.api.reader.DataLoader; +import org.labkey.api.reader.DataLoaderFactory; +import org.labkey.api.util.DateUtil; +import org.labkey.api.util.FileType; +import org.labkey.api.util.FileUtil; +import org.labkey.api.webdav.WebdavResource; +import org.labkey.api.webdav.WebdavService; +import org.labkey.signaldata.assay.SignalDataAssayDataHandler; +import org.labkey.vfs.FileLike; +import org.labkey.vfs.FileSystemLike; + +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.net.URI; +import java.net.URLDecoder; +import java.nio.charset.StandardCharsets; +import java.nio.file.Path; +import java.time.LocalDateTime; +import java.time.format.DateTimeFormatter; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; + +import static org.labkey.api.files.FileContentService.UPLOADED_FILE; + +public class SignalDataImportTask extends PipelineJob.Task +{ + public static final String PROTOCOL_NAME_PROPERTY = "protocolName"; + + // metadata file column names + private static final String INPUT_NAME = "Name"; + private static final String INPUT_DATA_FILE = "DataFile"; + + private String _folderName; + + private SignalDataImportTask(SignalDataImportTask.Factory factory, PipelineJob job) + { + super(factory, job); + } + + @NotNull + @Override + public RecordedActionSet run() + { + PipelineJob job = getJob(); + FileAnalysisJobSupport support = job.getJobSupport(FileAnalysisJobSupport.class); + job.setLogFile(support.getDataDirectory().resolveChild(FileUtil.makeFileNameWithTimestamp("triggered_signaldata_import", "log"))); + job.setStatus("RELOADING", "Job started at: " + DateUtil.nowISO()); + Logger log = job.getLogger(); + Container container = job.getContainer(); + + // validate the protocol + String protocolName = job.getParameters().get(PROTOCOL_NAME_PROPERTY); + if (StringUtils.isBlank(protocolName)) + { + log.error("Protocol name cannot be blank"); + return new RecordedActionSet(); + } + + ExpProtocol protocol = AssayService.get().getAssayProtocolByName(container, protocolName); + if (protocol == null) + { + log.error("Could not resolve the specified protocol name : {}", protocolName); + return new RecordedActionSet(); + } + + AssayProvider provider = AssayService.get().getProvider(protocol); + if (provider == null) + { + log.error("No Assay provider found for protocol name : {}", protocolName); + return new RecordedActionSet(); + } + + // guaranteed to only have a single file + if (support.getInputFiles().size() != 1) + { + log.error("Expecting a single input file but received {}", support.getInputFiles().size()); + return new RecordedActionSet(); + } + FileLike dataFile = support.getInputFiles().getFirst(); + + try + { + FileLike runRoot = getTargetFolder(container, log); + if (runRoot == null) + return new RecordedActionSet(); + + log.info("Loading {}", dataFile.getName()); + List> dataRows = parseMetadata(dataFile, log); + List> dataInputs = new ArrayList<>(); + + for (Map row : dataRows) + { + // parse out the name and datafile properties + String name = Objects.toString(row.get(INPUT_NAME), "").trim(); + String dataFilePath = Objects.toString(row.get(INPUT_DATA_FILE), "").trim(); + + // validate the existence of the name and datafile property and make a copy to the run root + if (StringUtils.isBlank(name)) + { + log.warn("Skipping row with blank Name property"); + continue; + } + + if (StringUtils.isBlank(dataFilePath)) + { + log.warn("Skipping row '{}' with blank DataFile property", name); + continue; + } + + // If the value is just a filename (no directory separators), resolve it relative to + // the metadata file's directory; otherwise treat it as a full server-side path + String dataFileName = FileUtil.getFileName(Path.of(dataFilePath)); + FileLike sourceFile = null; + if (dataFilePath.equals(dataFileName)) + { + String sourcePath = support.getParameters().get(DataTransformService.ORIGINAL_SOURCE_PATH); + if (StringUtils.isNotBlank(sourcePath)) + { + FileLike originalSource = FileSystemLike.wrapFile(new File(sourcePath)); + sourceFile = originalSource.getParent().resolveChild(dataFilePath); + } + } + else + { + // check to see if it's a webdav url + WebdavResource resource = WebdavService.get().lookup(dataFilePath); + if (resource != null) + { + sourceFile = FileSystemLike.wrapFile(resource.getFile()); + } + + // check to see if it's a server-side path + if (sourceFile == null) + { + Path resolvedPath = Path.of(dataFilePath).toAbsolutePath().normalize(); + + if (!isUnderAnyPipelineRoot(resolvedPath)) + { + log.error("DataFile '{}' is not under a server-managed pipeline root", dataFilePath); + row.remove(INPUT_DATA_FILE); + continue; + } + sourceFile = FileSystemLike.wrapFile(resolvedPath.toFile()); + } + } + + if (sourceFile != null && !sourceFile.exists()) + { + log.info("Data file not found: {}", sourceFile.getPath()); + row.remove(INPUT_DATA_FILE); + continue; + } + + // add a data input entry for the run + Map dataInput = new CaseInsensitiveHashMap<>(); + dataInputs.add(dataInput); + + log.info("Copying {} to run folder", sourceFile.getName()); + FileLike destFile = runRoot.resolveChild(sourceFile.getName()); + FileUtil.copyFile(sourceFile, destFile); + + log.info("Ensuring input data is created for {}", destFile.getName()); + URI uri = FileContentService.get().getWebDavUrl(destFile, container, FileContentService.PathType.full); + if (uri != null) + { + WebdavResource resource = WebdavService.get().lookup(uri.getPath()); + if (resource != null) + { + ExpData data = FileContentService.get().getDataObject(resource, container); + if (data == null) + { + // create the ExpData object for the input data + data = ExperimentService.get().createData(container, UPLOADED_FILE); + data.setName(destFile.getName()); + data.setDataFileURI(destFile.toURI()); + data.save(job.getUser()); + } + + FileLike d = FileUtil.getAbsoluteCaseSensitiveFile(destFile); + String url = d.toURI().toURL().toString(); + + dataInput.put(ExpDataTable.Column.Name.name(), data.getName()); + dataInput.put(ExpDataTable.Column.DataFileUrl.name(), data.getDataFileUrl()); + + // file data type for this run data field, adjust the URL to be compatible + String dataFileUrl = URLDecoder.decode(url, StandardCharsets.UTF_8); + row.replace(INPUT_DATA_FILE, dataFileUrl.replace("file:", "")); + } + else + log.warn("Unable to locate the webdav resource at {}", uri.getPath()); + } + else + log.warn("Unable to resolve a webdav URL for {}", destFile.getName()); + } + + // create and save the run + if (!dataRows.isEmpty()) + { + AssayRunUploadContext.Factory runFactory = provider.createRunUploadFactory(protocol, job.getUser(), container); + + runFactory.setName(_folderName); + runFactory.setLogger(log); + runFactory.setRawData(MapDataIterator.of(dataRows)); + runFactory.setRunProperties(Map.of("RunIdentifier", _folderName)); + + Map inputDatasMap = new HashMap<>(); + for (Map inputMap : dataInputs) + { + String dataFileUrl = Objects.toString(inputMap.get(ExpDataTable.Column.DataFileUrl.name()), ""); + if (!dataFileUrl.isEmpty()) + { + ExpData expData = ExperimentService.get().getExpDataByURL(dataFileUrl, container); + if (expData != null) + inputDatasMap.put(expData, Objects.toString(inputMap.get(ExpDataTable.Column.Name.name()), "")); + } + } + if (!inputDatasMap.isEmpty()) + runFactory.setInputDatas(inputDatasMap); + + // generate output data + Map outputData = new HashMap<>(); + DefaultAssayRunCreator.generateResultData(job.getUser(), container, provider, dataRows, outputData, log); + runFactory.setOutputDatas(outputData); + + try + { + provider.getRunCreator().saveExperimentRun(runFactory.create(), null); + } + catch (ValidationException | ExperimentException e) + { + log.error("Error saving assay run: {}", e.getMessage(), e); + throw new RuntimeException(e); + } + } + } + catch (Exception e) + { + log.error("Error importing data : {}", e.getMessage()); + throw new RuntimeException(e); + } + + return new RecordedActionSet(); + } + + private List> parseMetadata(FileLike dataFile, Logger log) + { + DataLoaderFactory dlf = DataLoader.get().findFactory(dataFile, null); + if (null == dlf) + { + log.error("Unable to find a loader for file : {}", dataFile.getPath()); + return Collections.emptyList(); + } + + try (InputStream in = dataFile.openInputStream(); + DataLoader loader = dlf.createLoader(in, true)) + { + return loader.load(); + } + catch (Exception e) + { + log.error("Error parsing the metadata file : {}", e.getMessage()); + return Collections.emptyList(); + } + } + + @Nullable + private FileLike getTargetFolder(Container container, Logger log) throws IOException + { + PipeRoot root = PipelineService.get().findPipelineRoot(container); + if (root != null) + { + _folderName = LocalDateTime.now() + .format(DateTimeFormatter.ofPattern("yyyy_M_d_H_m_s")); + + //Create folder if needed + FileLike runRoot = root.getRootFileLike().resolveChild(SignalDataAssayDataHandler.NAMESPACE).resolveChild(_folderName); + if (!runRoot.exists()) + runRoot.mkdirs(); + + return runRoot; + } + else + log.error("Unable to find a pipeline root for container : {}", container.getPath()); + + return null; + } + + /** + * Determine whether the given path falls under a pipeline root for some container, using the same semantics as + * {@link PipelineService#findPipelineRoot(Container)} (which includes the default file-root fallback, not just + * explicitly configured pipeline roots). First try to resolve the path directly to its owning container(s); if + * that comes up empty (e.g. a container with a custom, non-default file root that the path-resolution logic does + * not yet handle), fall back to scanning every container's pipeline root. + */ + private boolean isUnderAnyPipelineRoot(Path resolvedPath) + { + for (Container c : FileContentService.get().getContainersForFilePath(resolvedPath)) + { + if (isUnderPipelineRoot(c, resolvedPath)) + return true; + } + + // Path could not be resolved to a container directly; fall back to scanning all containers + for (Container c : ContainerManager.getAllChildren(ContainerManager.getRoot())) + { + if (isUnderPipelineRoot(c, resolvedPath)) + return true; + } + + return false; + } + + private boolean isUnderPipelineRoot(Container container, Path resolvedPath) + { + PipeRoot root = PipelineService.get().findPipelineRoot(container); + return root != null && root.isUnderRoot(resolvedPath); + } + + public static class Factory extends AbstractTaskFactory + { + public Factory() + { + super(SignalDataImportTask.class); + } + + @Override + public SignalDataImportTask createTask(PipelineJob job) + { + return new SignalDataImportTask(this, job); + } + + @Override + public List getInputTypes() + { + return Collections.emptyList(); + } + + @Override + public List getProtocolActionNames() + { + return Collections.emptyList(); + } + + @Override + public String getStatusName() + { + return "IMPORT SIGNAL DATA"; + } + + @Override + public boolean isJobComplete(PipelineJob job) + { + return false; + } + } +} diff --git a/signalData/test/src/org/labkey/test/tests/signaldata/SignalDataFileWatcherTest.java b/signalData/test/src/org/labkey/test/tests/signaldata/SignalDataFileWatcherTest.java new file mode 100644 index 000000000..03be71537 --- /dev/null +++ b/signalData/test/src/org/labkey/test/tests/signaldata/SignalDataFileWatcherTest.java @@ -0,0 +1,289 @@ +/* + * Copyright (c) 2016-2026 LabKey Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.labkey.test.tests.signaldata; + +import org.jetbrains.annotations.Nullable; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.labkey.api.util.FileUtil; +import org.labkey.api.util.Path; +import org.labkey.test.BaseWebDriverTest; +import org.labkey.test.Locator; +import org.labkey.test.categories.Daily; +import org.labkey.test.components.pipeline.PipelineTriggerWizard; +import org.labkey.test.pages.signaldata.SignalDataAssayBeginPage; +import org.labkey.test.util.DataRegionTable; +import org.labkey.test.util.PipelineStatusTable; +import org.labkey.test.util.PortalHelper; +import org.labkey.test.util.PostgresOnlyTest; +import org.labkey.test.util.core.webdav.WebDavUploadHelper; +import org.labkey.test.util.data.TestDataUtils; +import org.labkey.test.util.query.QueryUtils; +import org.labkey.test.util.signaldata.SignalDataInitializer; + +import java.io.File; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +@Category({Daily.class}) +@BaseWebDriverTest.ClassTimeout(minutes = 10) +public class SignalDataFileWatcherTest extends BaseWebDriverTest implements PostgresOnlyTest +{ + private static final String PROJECT_NAME = "SignalDataFileWatcherTest"; + private static final String DEFAULT_RUN = "TestRun001"; + private static final String ASSAY_DATA_LOC = "SignalDataAssayData/" + DEFAULT_RUN; + + // The pipeline description registered for the import task in signaldataContext.xml. It is the link text on the + // folder management Import tab and the label of the trigger's task selector. + private static final String IMPORT_PIPELINE_TASK = "Import Signal Data run from a Metadata File"; + + // Data files referenced by RunsMetadata/datafiles.tsv (by bare file name). + private static final String RESULT_FILENAME_1 = "LGC12392.TXT"; + private static final String RESULT_FILENAME_2 = "LGC14332.TXT"; + private static final String RESULT_FILENAME_3 = "MPP82113.TXT"; + + // A file root subdirectory, separate from where metadata files are dropped, used to exercise data files + // referenced by WebDAV path rather than by bare name. + private static final String ALT_DATA_DIR = "altData"; + + @Nullable + @Override + protected final String getProjectName() + { + return PROJECT_NAME; + } + + @Override + public List getAssociatedModules() + { + return Collections.singletonList("SignalData"); + } + + @BeforeClass + public static void doSetup() throws Exception + { + SignalDataFileWatcherTest test = getCurrentTest(); + SignalDataInitializer initializer = new SignalDataInitializer(test, test.getProjectName()); + initializer.setupProject(); + + // The file watcher drops files into the project file root; expose a file browser there for the upload. + test.goToProjectHome(); + new PortalHelper(test.getDriver()).addBodyWebPart("Files"); + + // Place the data files in two locations: the file root, where the metadata file will be dropped (so rows + // that reference data files by bare name resolve relative to the metadata file's directory), and a separate + // subdirectory referenced by WebDAV path in testWebDavDataFilePaths. + WebDavUploadHelper uploadHelper = new WebDavUploadHelper(test.getPrimaryTestProject()); + uploadHelper.mkDir(ALT_DATA_DIR); + for (String dataFile : List.of(RESULT_FILENAME_1, RESULT_FILENAME_2, RESULT_FILENAME_3)) + { + File file = test.getFile(ASSAY_DATA_LOC + "/" + dataFile); + uploadHelper.uploadFile(file, ""); + uploadHelper.uploadFile(file, ALT_DATA_DIR); + } + } + + @Before + public void preTest() throws Exception + { + // Each test imports a new run (the import task names runs with a generated timestamp). Delete any runs from + // a previous test, keeping only the run created during setup, so the imported run can be identified by + // exclusion. Also clear trigger configurations and completed jobs between tests. + log("Reset runs, trigger configurations, and completed jobs"); + navigateToAssayLandingPage(SignalDataInitializer.RAW_SignalData_ASSAY).resetUploadedData(DEFAULT_RUN); + QueryUtils.truncateTable(getProjectName(), "pipeline", "TriggerConfigurations"); + deleteAllPipelineJobs(); + } + + @Test + public void testMetadataFileWatcherImport() + { + File metadataFile = getFile("RunsMetadata/datafiles.tsv"); + + log("Configure a file watcher trigger for the Signal Data import pipeline"); + createImportTrigger("Signal Data import trigger", metadataFile.getName()); + + log("Drop the metadata file into the watched file root to trigger the import"); + goToProjectHome(); + _fileBrowserHelper.dragDropUpload(metadataFile); + + log("Wait for the file watcher to run the import job"); + goToDataPipeline(); + waitForPipelineJobsToFinish(2); + + log("Verify a new run was imported from the metadata file"); + SignalDataAssayBeginPage beginPage = navigateToAssayLandingPage(SignalDataInitializer.RAW_SignalData_ASSAY); + String importedRun = getImportedRunIdentifier(beginPage); + + beginPage.setSearchBox(importedRun); + assertEquals("Incorrect number of rows imported by the file watcher", 3, beginPage.getRowCount()); + + DataRegionTable table = beginPage.getDataRegionTable(); + assertArrayEquals("Incorrect Name values for the imported run", + new String[]{RESULT_FILENAME_1, RESULT_FILENAME_2, RESULT_FILENAME_3}, + table.getColumnDataAsText("Name").toArray()); + assertArrayEquals("Incorrect StringValue values for the imported run", + new String[]{"StringOne", "StringTwo", "StringThree"}, + table.getColumnDataAsText("StringValue").toArray()); + assertArrayEquals("Incorrect IntegerValue values for the imported run", + new String[]{"1", "2", "3"}, + table.getColumnDataAsText("IntegerValue").toArray()); + } + + @Test + public void testWebDavDataFilePaths() throws IOException + { + List> rows = new ArrayList<>(); + rows.add(List.of("Name", "DataFile", "StringValue", "IntegerValue")); + rows.add(List.of(RESULT_FILENAME_1, webDavPath(RESULT_FILENAME_1), "StringOne", "1")); + rows.add(List.of(RESULT_FILENAME_2, webDavPath(RESULT_FILENAME_2), "StringTwo", "2")); + rows.add(List.of(RESULT_FILENAME_3, webDavPath(RESULT_FILENAME_3), "StringThree", "3")); + File metadataFile = TestDataUtils.writeRowsToTsv("webdavDatafiles.tsv", rows); + + log("Configure a file watcher trigger for the Signal Data import pipeline"); + createImportTrigger("WebDav paths trigger", metadataFile.getName()); + + log("Drop the metadata file; its data files are referenced by WebDAV path in a separate folder"); + goToProjectHome(); + _fileBrowserHelper.dragDropUpload(metadataFile); + + log("Wait for the file watcher to run the import job"); + goToDataPipeline(); + waitForPipelineJobsToFinish(2); + + log("Verify a new run was imported from the WebDAV-referenced data files"); + SignalDataAssayBeginPage beginPage = navigateToAssayLandingPage(SignalDataInitializer.RAW_SignalData_ASSAY); + String importedRun = getImportedRunIdentifier(beginPage); + + beginPage.setSearchBox(importedRun); + assertEquals("Incorrect number of rows imported by the file watcher", 3, beginPage.getRowCount()); + + DataRegionTable table = beginPage.getDataRegionTable(); + assertArrayEquals("Incorrect Name values for the imported run", + new String[]{RESULT_FILENAME_1, RESULT_FILENAME_2, RESULT_FILENAME_3}, + table.getColumnDataAsText("Name").toArray()); + assertArrayEquals("Incorrect StringValue values for the imported run", + new String[]{"StringOne", "StringTwo", "StringThree"}, + table.getColumnDataAsText("StringValue").toArray()); + assertArrayEquals("Incorrect IntegerValue values for the imported run", + new String[]{"1", "2", "3"}, + table.getColumnDataAsText("IntegerValue").toArray()); + } + + @Test + public void testDataFileOutsidePipelineRootIsRejected() throws IOException + { + String outsidePath = "/not/a/pipeline/root/OutsideRoot.TXT"; + List> rows = new ArrayList<>(); + rows.add(List.of("Name", "DataFile")); + rows.add(List.of(RESULT_FILENAME_1, RESULT_FILENAME_1)); + rows.add(List.of("OutsideRoot.TXT", outsidePath)); + File metadataFile = TestDataUtils.writeRowsToTsv("outsideRoot.tsv", rows); + + log("Configure a file watcher trigger for the Signal Data import pipeline"); + createImportTrigger("Outside root trigger", metadataFile.getName()); + + log("Drop a metadata file referencing a data file outside any pipeline root"); + goToProjectHome(); + _fileBrowserHelper.dragDropUpload(metadataFile); + + log("Wait for the file watcher import job to finish"); + goToDataPipeline(); + waitForPipelineJobsToFinish(2); + + log("Verify the import job logged the pipeline-root rejection"); + PipelineStatusTable statusTable = new PipelineStatusTable(getDriver()); + List statuses = statusTable.getColumnDataAsText("Status"); + int errorRow = statuses.indexOf("ERROR"); + assertTrue("Expected one of the pipeline jobs to be in error, statuses were: " + statuses, errorRow >= 0); + + statusTable.clickStatusLink(errorRow) + .waitForError(String.format("DataFile '%s' is not under a server-managed pipeline root", outsidePath)); + + // The rejection is logged as an error and surfaces in the server error log; account for it so the harness's + // post-test error check does not fail this test. + deleteAllPipelineJobs(); + checkExpectedErrors(1); + } + + private void createImportTrigger(String name, String filePattern) + { + goToProjectHome(); + goToFolderManagement().goToImportTab(); + waitAndClickAndWait(Locator.linkWithText(IMPORT_PIPELINE_TASK)); + + PipelineTriggerWizard wizard = new PipelineTriggerWizard(getDriver()); + wizard.setName(name) + .setTask(IMPORT_PIPELINE_TASK) + .setEnabled(true) + .goToConfiguration() + .setLocation(".") + .setFilePattern(filePattern) + // The 'protocolName' custom field declared by SignalDataImportTask's pipeline; the wizard binds + // setAssayProtocol() to the input named "protocolName". + .setAssayProtocol(SignalDataInitializer.RAW_SignalData_ASSAY); + wizard.saveConfiguration(); + } + + /** + * Returns the single run identifier present on the assay landing page other than the run created during setup. + * The import task names the run with a generated timestamp, so it is identified by exclusion. + */ + private String getImportedRunIdentifier(SignalDataAssayBeginPage beginPage) + { + List importedRuns = beginPage.getDataRegionTable().getColumnDataAsText("Run Identifier").stream() + .filter(runId -> !DEFAULT_RUN.equals(runId)) + .distinct() + .toList(); + assertEquals("Expected exactly one newly imported run, found: " + importedRuns, 1, importedRuns.size()); + return importedRuns.get(0); + } + + /** + * Builds the server-relative WebDAV resource path for a data file in the alternate data directory, in the form + * resolved by {@code WebdavService.lookup}: {@code /_webdav//@files//}. The project name + * has no characters that require encoding. + */ + private String webDavPath(String fileName) + { + return String.format("/_webdav/%s/@files/%s/%s", getProjectName(), ALT_DATA_DIR, fileName); + } + + private File getFile(String relativePath) + { + File file = FileUtil.appendPath(SignalDataInitializer.RAW_SignalData_SAMPLE_DATA, Path.parse(relativePath)); + if (!file.exists()) + throw new RuntimeException("Can't find path: " + file.getAbsolutePath()); + return file; + } + + private SignalDataAssayBeginPage navigateToAssayLandingPage(String assayName) + { + goToProjectHome(); + clickAndWait(Locator.linkWithText(assayName)); + SignalDataAssayBeginPage page = new SignalDataAssayBeginPage(this); + page.waitForPageLoad(); + return page; + } +} diff --git a/signalData/webapp/WEB-INF/signalData/signaldataContext.xml b/signalData/webapp/WEB-INF/signalData/signaldataContext.xml new file mode 100644 index 000000000..0d1af2808 --- /dev/null +++ b/signalData/webapp/WEB-INF/signalData/signaldataContext.xml @@ -0,0 +1,62 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + org.labkey.signaldata.pipeline.SignalDataImportTask + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file