Use GTFS directions file when provided #lio

src/main/java/org/mtransit/parser/DefaultAgencyTools.java

@@ -1,5 +1,7 @@
 package org.mtransit.parser;
 
+import static org.mtransit.parser.Constants.EMPTY;
+
 import androidx.annotation.Discouraged;
 import androidx.annotation.VisibleForTesting;
 
@@ -67,7 +69,7 @@
 public class DefaultAgencyTools implements GAgencyTools {
 
 	static {
-		CommonsApp.setup(false);
+		CommonsApp.setup(false); // isAndroidPlatform
 	}
 
 	@SuppressWarnings("WeakerAccess")
@@ -871,9 +873,10 @@ public void setDirectionHeadsign(@NotNull MRoute mRoute, @NotNull MDirection mDi
 		}
 		final boolean fromStopName = mDirection.getHeadsignType() == MDirection.HEADSIGN_TYPE_STOP_ID;
 		final boolean fromTripHeadSign = mDirection.getHeadsignType() == MDirection.HEADSIGN_TYPE_STRING;
+		final boolean fromDirection = false; // using trip head-sign here
 		if (directionFinderEnabled(mRoute.getId(), gRoute)) {
 			mDirection.setHeadsignString(
-					cleanDirectionHeadsign(gRoute, gTrip.getDirectionIdOrDefault(), fromStopName, fromTripHeadSign, gTrip.getTripHeadsignOrDefault()),
+					cleanDirectionHeadsign(gRoute, gTrip.getDirectionIdOrDefault(), fromStopName, fromTripHeadSign, fromDirection, gTrip.getTripHeadsignOrDefault()),
 					gTrip.getDirectionIdOrDefault()
 			);
 			return;
@@ -883,7 +886,7 @@ public void setDirectionHeadsign(@NotNull MRoute mRoute, @NotNull MDirection mDi
 		}
 		try {
 			mDirection.setHeadsignString(
-					cleanDirectionHeadsign(gRoute, gTrip.getDirectionIdOrDefault(), fromStopName, fromTripHeadSign, gTrip.getTripHeadsignOrDefault()),
+					cleanDirectionHeadsign(gRoute, gTrip.getDirectionIdOrDefault(), fromStopName, fromTripHeadSign, fromDirection, gTrip.getTripHeadsignOrDefault()),
 					gTrip.getDirectionIdOrDefault()
 			);
 		} catch (NumberFormatException nfe) {
@@ -997,10 +1000,25 @@ public boolean removeRouteDescFromDirectionHeadsign() {
 	 * @param directionId {@link org.mtransit.parser.gtfs.data.GDirectionId} (0 or 1 or missing/generated)
 	 */
 	@Override
+	public @NotNull String cleanDirectionHeadsign(@Nullable GRoute gRoute, int directionId, boolean fromStopName, boolean fromTripHeadSign, boolean fromDirection, @NotNull String directionHeadSign) {
+		if (fromDirection) {
+			//noinspection DiscouragedApi
+			final String originalRouteId = gRoute == null ? null : gRoute.getOriginalRouteId();
+			if (Configs.getRouteConfig().isDirectionHeadsignIgnoreProvidedDirection(getTodayDateInt(), originalRouteId)) {
+				return EMPTY;
+			}
+		}
+		//noinspection deprecation
+		return cleanDirectionHeadsign(gRoute, directionId, fromStopName, fromTripHeadSign, directionHeadSign);
+	}
+
+	@SuppressWarnings("DeprecatedIsStillUsed")
+	@Deprecated
+	@Override
 	public @NotNull String cleanDirectionHeadsign(@Nullable GRoute gRoute, int directionId, boolean fromStopName, boolean fromTripHeadSign, @NotNull String directionHeadSign) {
 		if (fromTripHeadSign) {
 			if (Configs.getRouteConfig().getDirectionHeadsignIgnoreTripHeadsign()) {
-				directionHeadSign = "";
+				return EMPTY;
 			}
 		}
 		//noinspection deprecation
@@ -1013,26 +1031,26 @@ public boolean removeRouteDescFromDirectionHeadsign() {
 	public @NotNull String cleanDirectionHeadsign(@Nullable GRoute gRoute, int directionId, boolean fromStopName, @NotNull String directionHeadSign) {
 		if (gRoute != null) {
 			if (removeRouteLongNameFromDirectionHeadsign() && directionHeadSign.equals(gRoute.getRouteLongNameOrDefault())) {
-				directionHeadSign = "";
+				return EMPTY;
 			} else if (removeRouteShortNameFromDirectionHeadsign() && directionHeadSign.equals(gRoute.getRouteShortName())) {
-				directionHeadSign = "";
+				return EMPTY;
 			} else if (removeRouteDescFromDirectionHeadsign() && directionHeadSign.equals(gRoute.getRouteDescOrDefault())) {
-				directionHeadSign = "";
+				return EMPTY;
 			}
 		}
 		//noinspection deprecation
-		directionHeadSign = cleanDirectionHeadsign(directionId, fromStopName, directionHeadSign);
-		return directionHeadSign;
+		return cleanDirectionHeadsign(directionId, fromStopName, directionHeadSign);
 	}
 
 	@SuppressWarnings("DeprecatedIsStillUsed")
 	@Deprecated
 	@NotNull
 	@Override
 	public String cleanDirectionHeadsign(int directionId, boolean fromStopName, @NotNull String directionHeadSign) {
-		directionHeadSign = Configs.getRouteConfig().cleanDirectionHeadsign(getFirstLanguageNN(), directionHeadSign);
 		//noinspection deprecation
-		return cleanDirectionHeadsign(fromStopName, directionHeadSign);
+		return cleanDirectionHeadsign(fromStopName,
+				Configs.getRouteConfig().cleanDirectionHeadsign(getFirstLanguageNN(), directionHeadSign)
+		);
 	}
 
 	@SuppressWarnings("DeprecatedIsStillUsed")

src/main/java/org/mtransit/parser/FileUtils.java

@@ -6,6 +6,7 @@
 import java.io.File;
 import java.io.IOException;
 import java.nio.file.Files;
+import java.util.List;
 
 @SuppressWarnings("unused")
 public final class FileUtils {
@@ -55,7 +56,7 @@ public static String sizeToDiplayString(@Nullable Long size) {
 	}
 
 	@Nullable
-	public static File findFileCaseInsensitive(@NotNull String directoryPath, @NotNull String fileNameToFind) {
+	public static File findFileCaseInsensitive(@NotNull String directoryPath, @NotNull List<String> fileNamesToFind) {
 		final File directory = new File(directoryPath);
 		// Check if the provided path is a directory and exists
 		if (!directory.isDirectory()) {
@@ -64,9 +65,9 @@ public static File findFileCaseInsensitive(@NotNull String directoryPath, @NotNu
 		}
 		final File[] files = directory.listFiles(); // Get all files and subdirectories
 		if (files != null) {
-			for (File file : files) {
-				if (file.isFile()) { // Check if it's a file (not a directory)
-					if (file.getName().equalsIgnoreCase(fileNameToFind)) {
+			for (String fileNameToFind : fileNamesToFind) {
+				for (File file : files) {
+					if (file.isFile() && file.getName().equalsIgnoreCase(fileNameToFind)) {
 						return file; // Found the file (case-insensitive)
 					}
 				}

src/main/java/org/mtransit/parser/config/gtfs/data/RouteConfig.kt

@@ -92,6 +92,10 @@ data class RouteConfig(
     val directionHeadsignRemoveRouteShortName: Boolean = false, // OPT-IN feature
     @SerialName("direction_headsign_remove_route_desc")
     val directionHeadsignRemoveRouteDesc: Boolean = false, // OPT-IN feature
+    @SerialName("direction_headsign_ignore_provided_direction")
+    val directionHeadsignIgnoreProvidedDirection: Boolean = false, // OPT-IN feature
+    @SerialName("direction_headsign_ignore_provided_direction_until")
+    val directionHeadsignIgnoreProvidedDirectionUntil: Map<String, String> = emptyMap(), // OPT-IN feature
     @SerialName("direction_headsign_ignore_trip_headsign")
     val directionHeadsignIgnoreTripHeadsign: Boolean = false, // OPT-IN feature
     @SerialName("direction_splitter_enabled")
@@ -453,6 +457,10 @@ data class RouteConfig(
         return string
     }
 
+    @JvmOverloads
+    fun isDirectionHeadsignIgnoreProvidedDirection(todayDate: Int, gOriginalRouteId: String? = null) =
+        directionHeadsignIgnoreProvidedDirection || isAllowedUntil(directionHeadsignIgnoreProvidedDirectionUntil[gOriginalRouteId], todayDate)
+
     @JvmOverloads
     fun isDirectionSplitterEnabled(todayDate: Int, routeId: Long? = null) =
         directionSplitterEnabled && !isAllowedUntil(directionSplitterDisabledUntil[routeId], todayDate)
@@ -470,6 +478,6 @@ data class RouteConfig(
     fun allowNonDescriptiveHeadSigns(todayDate: Int, mRouteId: Long) =
         allowNonDescriptiveHeadSigns[mRouteId] == true || isAllowedUntil(allowNonDescriptiveHeadSignsUntil[mRouteId], todayDate)
 
-    private fun isAllowedUntil(dateStr: String?, todayDate: Int) =
+    fun isAllowedUntil(dateStr: String?, todayDate: Int) =
         dateStr?.toIntOrNull()?.let { todayDate <= it } ?: false
 }

src/main/java/org/mtransit/parser/gtfs/GAgencyTools.java

@@ -214,6 +214,10 @@ public interface GAgencyTools {
 
 	boolean removeRouteDescFromDirectionHeadsign();
 
+	@NotNull
+	String cleanDirectionHeadsign(@Nullable GRoute gRoute, int directionId, boolean fromStopName, boolean fromTripHeadSign, boolean fromDirection, @NotNull String directionHeadSign);
+
+	@Deprecated
 	@NotNull
 	String cleanDirectionHeadsign(@Nullable GRoute gRoute, int directionId, boolean fromStopName, boolean fromTripHeadSign, @NotNull String directionHeadSign);
 

src/main/java/org/mtransit/parser/gtfs/GReader.java

@@ -36,6 +36,7 @@
 import java.sql.PreparedStatement;
 import java.text.DateFormat;
 import java.util.Arrays;
+import java.util.Collections;
 import java.util.Date;
 import java.util.HashMap;
 import java.util.HashSet;
@@ -125,7 +126,7 @@ public static GSpec readGtfsZipFile(
 			}
 			// DIRECTIONS (ext) (after route)
 			if (!calendarsOnly && !routeTripCalendarsOnly) {
-				readFile(gtfsDir, GDirection.FILENAME, false, line ->
+				readFiles(gtfsDir, GDirection.getFILENAMES(), false, line ->
 						processDirection(agencyTools, gSpec, line, skipDataCleanup)
 				);
 			}
@@ -183,22 +184,53 @@ private static boolean readFile(
 			boolean fileRequired,
 			@NotNull LineProcessor lineProcessor
 	) {
-		return readFile(gtfsDir, fileName, fileRequired, lineProcessor, null);
+		return readFiles(
+				gtfsDir,
+				Collections.singletonList(fileName),
+				fileRequired,
+				lineProcessor
+		);
 	}
 
+	private static boolean readFiles(
+			@NotNull String gtfsDir,
+			@NotNull List<String> fileNames,
+			boolean fileRequired,
+			@NotNull LineProcessor lineProcessor
+	) {
+		return readFiles(gtfsDir, fileNames, fileRequired, lineProcessor, null);
+	}
+
+	@SuppressWarnings({"UnusedReturnValue", "SameParameterValue"})
 	private static boolean readFile(
 			@NotNull String gtfsDir,
 			@NotNull String fileName,
 			boolean fileRequired,
 			@NotNull LineProcessor lineProcessor,
 			@Nullable OnColumnNamesFound onColumnNamesFoundCallback
 	) {
-		final File gtfsFile = FileUtils.findFileCaseInsensitive(gtfsDir, fileName);
+		return readFiles(
+				gtfsDir,
+				Collections.singletonList(fileName),
+				fileRequired,
+				lineProcessor,
+				onColumnNamesFoundCallback
+		);
+	}
+
+	private static boolean readFiles(
+			@NotNull String gtfsDir,
+			@NotNull List<String> fileNames,
+			boolean fileRequired,
+			@NotNull LineProcessor lineProcessor,
+			@Nullable OnColumnNamesFound onColumnNamesFoundCallback
+	) {
+		final File gtfsFile = FileUtils.findFileCaseInsensitive(gtfsDir, fileNames);
 		if (gtfsFile == null || !gtfsFile.exists()) {
 			if (fileRequired) {
-				throw new MTLog.Fatal("'%s' file does not exist!", gtfsFile);
+				throw new MTLog.Fatal("'%s' file does not exist!", fileNames);
 			} else {
-				MTLog.log("Reading file '%s'... SKIP (non-existing).", fileName);
+				MTLog.log("Reading file(s) '%s'... SKIP (non-existing).", fileNames);
 				return false;
 			}
 		}
@@ -236,58 +268,55 @@ private static void readCsv(
 		String line;
 		String[] columnNames;
 		line = reader.readLine();
-		if (line == null || line.isEmpty()) {
-			return;
-		}
+		if (line == null || line.isEmpty()) return;
 		if (line.charAt(0) == '\uFEFF') { // remove 1st empty char
 			MTLog.log("Reading file '%s'... > remove 1st empty car", filename);
 			line = String.copyValueOf(line.toCharArray(), 1, line.length() - 1);
 		}
-		CSVRecord recordColumns = CSVParser.parse(line, CSV_FORMAT).getRecords().get(0);
-		columnNames = new String[recordColumns.size()];
-		for (int i = 0; i < recordColumns.size(); i++) {
-			columnNames[i] = recordColumns.get(i);
+		CSVRecord lineRecordColumns = CSVParser.parse(line, CSV_FORMAT).getRecords().get(0);
+		columnNames = new String[lineRecordColumns.size()];
+		for (int i = 0; i < lineRecordColumns.size(); i++) {
+			columnNames[i] = lineRecordColumns.get(i);
 		}
 		if (onColumnNamesFoundCallback != null) {
 			onColumnNamesFoundCallback.processColumnNames(Arrays.asList(columnNames));
 		}
-		if (columnNames.length == 0) {
-			return;
-		}
-		List<CSVRecord> records;
-		HashMap<String, String> map = new HashMap<>();
-		String[] lineColumns = new String[columnNames.length];
-		int recordColumnsSize;
+		if (columnNames.length == 0) return;
+		List<CSVRecord> lineRecords;
+		final HashMap<String, String> map = new HashMap<>();
 		int l = 0;
 		boolean withQuotes;
+		int warningCount = 0;
 		while ((line = reader.readLine()) != null) {
 			try {
 				try {
-					records = CSVParser.parse(line, CSV_FORMAT).getRecords();
+					lineRecords = CSVParser.parse(line, CSV_FORMAT).getRecords();
 					withQuotes = true;
 				} catch (Exception e) {
-					records = CSVParser.parse(line, CSV_FORMAT_NO_QUOTE).getRecords();
+					lineRecords = CSVParser.parse(line, CSV_FORMAT_NO_QUOTE).getRecords();
 					withQuotes = false;
 				}
-				if (records.isEmpty()) {
-					continue; // empty line
-				}
-				recordColumns = records.get(0);
-				recordColumnsSize = recordColumns.size();
-				if (columnNames.length != recordColumnsSize
-						&& columnNames.length != (recordColumnsSize + 1)) {
-					MTLog.log("File '%s' line invalid: %s columns instead of %s: %s", filename, recordColumnsSize, columnNames.length, line);
-					continue;
+				if (lineRecords.isEmpty()) continue; // empty line
+				lineRecordColumns = lineRecords.get(0);
+				// recordColumnsSize = lineRecordColumns.size();
+				if (lineRecordColumns.size() > columnNames.length) {
+					if (warningCount < 10) {
+						MTLog.log("File '%s' line contains MORE columns (%s:%s) than expected (%s:%s)!", filename, lineRecordColumns.size(), line, columnNames.length, Arrays.asList(columnNames));
+						warningCount++;
+					}
+				} else if (lineRecordColumns.size() < columnNames.length) {
+					if (warningCount < 10) {
+						MTLog.log("File '%s' line contains LESS columns (%s:%s) than expected (%s:%s)!", filename, lineRecordColumns.size(), line, columnNames.length, Arrays.asList(columnNames));
+						warningCount++;
+					}
 				}
-				for (int i = 0; i < lineColumns.length; i++) {
-					final String lineColumn = i >= recordColumns.size() ? EMPTY : recordColumns.get(i);
-					lineColumns[i] = withQuotes ?
+				map.clear();
+				for (int i = 0; i < columnNames.length; i++) {
+					String lineColumn = i < lineRecordColumns.size() ? lineRecordColumns.get(i) : EMPTY;
+					lineColumn = withQuotes ?
 							lineColumn :
 							QUOTE_.matcher(lineColumn).replaceAll(EMPTY);
-				}
-				map.clear();
-				for (int ci = 0; ci < recordColumnsSize; ++ci) {
-					map.put(columnNames[ci], lineColumns[ci]);
+					map.put(columnNames[i], lineColumn);
 				}
 				if (lineProcessor != null) {
 					lineProcessor.processLine(map);
@@ -443,7 +472,7 @@ private static void processDirection(GAgencyTools agencyTools, GSpec gSpec, Hash
 				logExclude("Exclude direction (!route): %s | %s.", gRoute == null ? null : gRoute.getRouteId(), gDirection.getDirectionId());
 				return;
 			}
-			final GDirection existingDirection = gSpec.getDirection(gDirection.getRouteIdInt(), gDirection.getDirectionId());
+			final GDirection existingDirection = gSpec.getRouteDirection(gDirection.getRouteIdInt(), gDirection.getDirectionId().getId());
 			if (existingDirection != null) {
 				//noinspection DiscouragedApi
 				MTLog.logDebug("Duplicate direction ID for route ID! (new:%s|old:%s)", gDirection.getDirectionId(), existingDirection.getDirectionId());