diff --git a/src/main/java/org/mtransit/parser/DefaultAgencyTools.java b/src/main/java/org/mtransit/parser/DefaultAgencyTools.java index b23120d..b0e16d1 100644 --- a/src/main/java/org/mtransit/parser/DefaultAgencyTools.java +++ b/src/main/java/org/mtransit/parser/DefaultAgencyTools.java @@ -1,6 +1,7 @@ package org.mtransit.parser; import static org.mtransit.parser.Constants.EMPTY; +import static org.mtransit.parser.gtfs.data.GSpecExtKt.fixMissingTripDirectionIds; import androidx.annotation.Discouraged; import androidx.annotation.VisibleForTesting; @@ -186,6 +187,7 @@ public void start(@NotNull String[] args) { final long start = System.currentTimeMillis(); final GSpec gtfs = GReader.readGtfsZipFile(args[0], this, false, false); MDataChangedManager.avoidCalendarDatesDataChanged(lastServiceDates, gtfs, this); + fixMissingTripDirectionIds(gtfs); gtfs.cleanupStops(); gtfs.cleanupExcludedData(); gtfs.cleanupStopTimesPickupDropOffTypes(this); diff --git a/src/main/java/org/mtransit/parser/config/gtfs/data/RouteConfig.kt b/src/main/java/org/mtransit/parser/config/gtfs/data/RouteConfig.kt index 5263b90..e60f728 100644 --- a/src/main/java/org/mtransit/parser/config/gtfs/data/RouteConfig.kt +++ b/src/main/java/org/mtransit/parser/config/gtfs/data/RouteConfig.kt @@ -77,6 +77,8 @@ data class RouteConfig( val tripIdNotUniqueAllowed: Boolean = false, // OPT-IN feature @SerialName("trip_id_clean_merged") val tripIdCleanMerged: Boolean = false, // OPT-IN feature + @SerialName("direction_id_use_other_trips_with_same_headsign") + val directionIdUseOtherTripsWithSameHeadsign: Boolean = false, // OPT-IN feature // DIRECTION @SerialName("direction_types") val directionTypes: List = emptyList(), diff --git a/src/main/java/org/mtransit/parser/gtfs/data/GSpec.java b/src/main/java/org/mtransit/parser/gtfs/data/GSpec.java index 2f617b9..3961301 100644 --- a/src/main/java/org/mtransit/parser/gtfs/data/GSpec.java +++ b/src/main/java/org/mtransit/parser/gtfs/data/GSpec.java @@ -34,8 +34,8 @@ public class GSpec { private static final boolean LOG_REMOVED = false; // private static final boolean LOG_REMOVED = true; // DEBUG - private static final boolean USE_DB_ONLY = false; - // private static final boolean USE_DB_ONLY = true; // WIP + protected static final boolean USE_DB_ONLY = false; + // protected static final boolean USE_DB_ONLY = true; // WIP @NotNull private final Map agenciesCache = new HashMap<>(); @@ -355,7 +355,7 @@ public void addTrip(@NotNull GTrip gTrip, @Nullable PreparedStatement insertStop } @NotNull - private Collection getAllTripRouteIdInts() { + protected Collection getAllTripRouteIdInts() { if (USE_DB_ONLY) { return GIDs.getInts(GTFSDataBase.selectTripRouteIds()); } @@ -369,25 +369,6 @@ public int readTripsCount() { return CollectionUtils.totalMapSize(this.routeIdIntTripsCache); } - public void updateTripDirectionId(@NotNull GDirectionId gDirectionId, @Nullable Collection tripIdInts) { - updateTripDirectionId(gDirectionId.getId(), tripIdInts); - } - - @SuppressWarnings("WeakerAccess") - public void updateTripDirectionId(int directionId, @Nullable Collection tripIdInts) { - if (tripIdInts == null) { - return; - } - List routeIdInts = new ArrayList<>(); - for (Integer tripIdInt : tripIdInts) { - routeIdInts.add(getTripRouteId(tripIdInt)); - } - for (Integer routeIdInt : routeIdInts) { - GTrip.updateDirectionIdForTrips(getRouteTrips(routeIdInt), tripIdInts, directionId); - GTFSDataBase.updateTrip(GIDs.getStrings(tripIdInts), directionId); - } - } - @Nullable public GTrip getTrip(int tripIdInt) { if (USE_DB_ONLY) { @@ -406,7 +387,7 @@ public GTrip getTrip(int tripIdInt) { } @Nullable - private Integer getTripRouteId(Integer tripIdInt) { + protected Integer getTripRouteId(int tripIdInt) { return this.tripIdIntRouteIdInt.get(tripIdInt); } @@ -471,10 +452,12 @@ public List getStopTimes(long mRouteId, int gTripIdInt) { } @NotNull - public List getStopTimes(long mRouteId, - int gTripIdInt, - @SuppressWarnings("unused") @Nullable String optGStopId, - @SuppressWarnings("unused") @Nullable Integer optGStopSequence) { + public List getStopTimes( + long mRouteId, + int gTripIdInt, + @SuppressWarnings("unused") @Nullable String optGStopId, + @SuppressWarnings("unused") @Nullable Integer optGStopSequence + ) { GenerateMObjectsTask routeGenerator = this.routeGenerators.get(mRouteId); if (routeGenerator != null) { return routeGenerator.getTripGStopTimes(gTripIdInt); @@ -796,9 +779,11 @@ public void generateStopTimesFromFrequencies(@SuppressWarnings("unused") @NotNul MTLog.log("- Stop times: %d (after) (new: %d)", readStopTimesCount(), st); } - private void setDepartureTimeCal(@NotNull Calendar calendar, - @NotNull GStopTime gStopTime, - @NotNull List tripStopTimes) { + private void setDepartureTimeCal( + @NotNull Calendar calendar, + @NotNull GStopTime gStopTime, + @NotNull List tripStopTimes + ) { if (gStopTime.hasDepartureTime()) { calendar.setTimeInMillis(gStopTime.getDepartureTimeMs()); } else { diff --git a/src/main/java/org/mtransit/parser/gtfs/data/GSpecExt.kt b/src/main/java/org/mtransit/parser/gtfs/data/GSpecExt.kt index e4816a8..0df9ef3 100644 --- a/src/main/java/org/mtransit/parser/gtfs/data/GSpecExt.kt +++ b/src/main/java/org/mtransit/parser/gtfs/data/GSpecExt.kt @@ -1,3 +1,48 @@ package org.mtransit.parser.gtfs.data +import org.mtransit.parser.MTLog +import org.mtransit.parser.config.Configs +import org.mtransit.parser.db.GTFSDataBase +import org.mtransit.parser.gtfs.data.GSpec.USE_DB_ONLY +import org.mtransit.parser.gtfs.data.GTrip.Companion.updateDirectionId + fun GSpec.getRoute(gTrip: GTrip) = this.getRoute(gTrip.routeIdInt) + +fun GSpec.updateTripDirectionId(directionId: Int, tripIdInts: Collection?) { + tripIdInts ?: return + if (!USE_DB_ONLY) { + tripIdInts + .mapNotNull { getTripRouteId(it) } + .distinct() + .forEach { routeIdInt -> + getRouteTrips(routeIdInt).updateDirectionId(tripIdInts, directionId) + } + } + GTFSDataBase.updateTrip(tripIds = GIDs.getStrings(tripIdInts), directionId) +} + +fun GSpec.fixMissingTripDirectionIds() { + if (!Configs.routeConfig.directionIdUseOtherTripsWithSameHeadsign) return + MTLog.log("Try fixing GTFS trips w/o direction IDs...") + var tripIdDirectionFixed = 0 + getAllTripRouteIdInts().forEach { routeIdInt -> + val gRouteTrips = getRouteTrips(routeIdInt) + gRouteTrips + .filter { it.directionIdOrOriginal == null } + .groupBy { it.tripHeadsign } + .forEach { (tripHeadsign, gTripsNoDirectionId) -> + if (tripHeadsign == null) return@forEach + val distinctOriginalDirectionIds = gRouteTrips + .asSequence() + .filter { it.tripHeadsign == tripHeadsign } + .mapNotNull { it.directionIdOrOriginal } + .distinct() + .toList() + distinctOriginalDirectionIds.singleOrNull()?.let { originalDirectionId -> + updateTripDirectionId(originalDirectionId, gTripsNoDirectionId.map { it.tripIdInt }) + tripIdDirectionFixed += gTripsNoDirectionId.size + } + } + } + MTLog.log("Try fixing GTFS trips w/o direction IDs... DONE (%d fixed trips with direction ID)", tripIdDirectionFixed) +} diff --git a/src/main/java/org/mtransit/parser/gtfs/data/GTrip.kt b/src/main/java/org/mtransit/parser/gtfs/data/GTrip.kt index 9fdd85b..3b08d43 100644 --- a/src/main/java/org/mtransit/parser/gtfs/data/GTrip.kt +++ b/src/main/java/org/mtransit/parser/gtfs/data/GTrip.kt @@ -46,8 +46,14 @@ data class GTrip( bikesAllowed = bikesAllowed, ) - val directionId: Int? - get() = directionIdE.originalId() // optional + @set:Discouraged(message = "Should not be changed") + var directionId: Int? + get() { + return directionIdE.originalId() // optional + } + private set(value) { + this.directionIdE = GDirectionId.parse(value) + } @Suppress("unused") val directionIdOrDefault: Int @@ -57,12 +63,6 @@ data class GTrip( val directionIdOrOriginal: Int? get() = directionIdE.originalId() - @Suppress("unused") - @Discouraged(message = "Should not be changed") - fun setDirectionId(newDirectionId: Int?) { - this.directionIdE = GDirectionId.parse(newDirectionId) - } - fun hasTripHeadsign() = !this.tripHeadsign.isNullOrBlank() @Suppress("unused") @@ -214,23 +214,10 @@ data class GTrip( } @JvmStatic - fun updateDirectionIdForTrips(gTrips: MutableList, tripIdInts: Collection, directionId: Int): List { - return updateList( - gTrips = gTrips, - condition = { tripIdInts.contains(it.tripIdInt) }, - updateTrip = { it.copy(directionIdE = GDirectionId.parse(directionId)) } - ) - } - - fun updateList(gTrips: MutableList, condition: (GTrip) -> Boolean, updateTrip: (GTrip) -> GTrip): List { - for (i in 0 until gTrips.size) { - val gTrip = gTrips[i] - if (condition(gTrip)) { - gTrips[i] = updateTrip(gTrip) - } + fun MutableList.updateDirectionId(tripIdInts: Collection, directionId: Int) = + replaceAll { gTrip -> + if (!tripIdInts.contains(gTrip.tripIdInt)) return@replaceAll gTrip + gTrip.copy(directionIdE = GDirectionId.parse(directionId)) } - - return gTrips - } } } \ No newline at end of file diff --git a/src/main/java/org/mtransit/parser/mt/MDirectionHeadSignFinder.kt b/src/main/java/org/mtransit/parser/mt/MDirectionHeadSignFinder.kt index db1a144..331097b 100644 --- a/src/main/java/org/mtransit/parser/mt/MDirectionHeadSignFinder.kt +++ b/src/main/java/org/mtransit/parser/mt/MDirectionHeadSignFinder.kt @@ -129,7 +129,7 @@ object MDirectionHeadSignFinder { val gRoute = directionRouteIdInts[directionId].takeIf { it?.size == 1 }?.getOrNull(0)?.let { routeGTFS.getRoute(it) } lastStopDirectionHeadSigns[directionId] = agencyTools.cleanDirectionHeadsign(gRoute, directionId, true, false, false, agencyTools.cleanStopName(stop.stopName)) - MTLog.logDebug("$routeId: $directionId Stop '${stop.toStringPlus(false)}' > '${lastStopDirectionHeadSigns[directionId]}'.") + MTLog.logDebug("$routeId: $directionId: - last stop '${stop.toStringPlus(false)}' > '${lastStopDirectionHeadSigns[directionId]}'.") } val allDirectionHeadSignsEmpty: Boolean = directionHeadSigns .map { (_, headSign) -> headSign } @@ -743,7 +743,7 @@ object MDirectionHeadSignFinder { MTLog.log( !dataLossAuthorized, "$routeId: $directionId: #2 goes for WAY more stops ($stopIdIntsAfterCommonCount2) than " + - "#1 ($stopIdIntsAfterCommonCount1:$stopTimesHeadSign1) " + + "#1 ($stopIdIntsAfterCommonCount1:'$stopTimesHeadSign1') " + "-> '$stopTimesHeadSign2'" ) return MergedTrip( @@ -762,7 +762,7 @@ object MDirectionHeadSignFinder { MTLog.log( !dataLossAuthorized, "$routeId: $directionId: #1 goes for WAY more stops ($stopIdIntsAfterCommonCount1) than " + - "#2 ($stopIdIntsAfterCommonCount2:$stopTimesHeadSign2) " + + "#2 ($stopIdIntsAfterCommonCount2:'$stopTimesHeadSign2') " + "-> '$stopTimesHeadSign1'" ) return MergedTrip( @@ -970,7 +970,7 @@ object MDirectionHeadSignFinder { !dataLossAuthorized, "$routeId: $directionId: distance from last common to " + "#1 last ($distanceToStop1) > " + - "#2 last ($distanceToStop2) ($stopTimesHeadSign2) " + + "#2 last ($distanceToStop2) ('$stopTimesHeadSign2') " + "-> '$stopTimesHeadSign1'" ) return MergedTrip( @@ -987,7 +987,7 @@ object MDirectionHeadSignFinder { !dataLossAuthorized, "$routeId: $directionId: distance from last common to " + "#2 last ($distanceToStop2) > " + - "#1 last ($distanceToStop1) ($stopTimesHeadSign1) " + + "#1 last ($distanceToStop1) ('$stopTimesHeadSign1') " + "-> '$stopTimesHeadSign2'" ) return MergedTrip( @@ -1006,18 +1006,18 @@ object MDirectionHeadSignFinder { if (dataLossAuthorized) { throw MTLog.Fatal( "$routeId: $directionId: Unresolved situation! \n" + - "- #1: $stopTimesHeadSign1. \n" + + "- #1: '$stopTimesHeadSign1'. \n" + " Stops: ${stopTimesList1.map { gStopTime -> "\n - ${gStopTime.toStringPlus(false)}" }} \n" + - "- #2: $stopTimesHeadSign2. \n" + + "- #2: '$stopTimesHeadSign2'. \n" + " Stops: ${stopTimesList2.map { gStopTime -> "\n - ${gStopTime.toStringPlus(false)}" }} \n" + "!" ) } logMerge( !dataLossAuthorized, "$routeId: $directionId: unresolved situation > no head-sign (" + - "$stopTimesHeadSign1:${GIDs.toStringPlus(stopTimesList1.firstOrNull()?.tripIdInt)}" + + "'$stopTimesHeadSign1':${GIDs.toStringPlus(stopTimesList1.firstOrNull()?.tripIdInt)}" + "|" + - "$stopTimesHeadSign2:${GIDs.toStringPlus(stopTimesList2.firstOrNull()?.tripIdInt)}" + + "'$stopTimesHeadSign2':${GIDs.toStringPlus(stopTimesList2.firstOrNull()?.tripIdInt)}" + ")." ) return null diff --git a/src/main/java/org/mtransit/parser/mt/MDirectionSplitter.kt b/src/main/java/org/mtransit/parser/mt/MDirectionSplitter.kt index 9c4012b..c94d0bc 100644 --- a/src/main/java/org/mtransit/parser/mt/MDirectionSplitter.kt +++ b/src/main/java/org/mtransit/parser/mt/MDirectionSplitter.kt @@ -13,6 +13,7 @@ import org.mtransit.parser.gtfs.data.GIDs import org.mtransit.parser.gtfs.data.GRoute import org.mtransit.parser.gtfs.data.GSpec import org.mtransit.parser.gtfs.data.GTrip +import org.mtransit.parser.gtfs.data.updateTripDirectionId import kotlin.math.min object MDirectionSplitter { @@ -60,8 +61,8 @@ object MDirectionSplitter { } if ((min(trips0Size, trips1Size).toFloat() / gRouteTrips.size) > .33f) { // TODO check if directions candidates group match existing split to keep original direction IDs - routeGTFS.updateTripDirectionId(GDirectionId.NEW_1, headSignToGTripIdInts[sortedHeadSigns[0]]) - routeGTFS.updateTripDirectionId(GDirectionId.NEW_2, headSignToGTripIdInts[sortedHeadSigns[1]]) + routeGTFS.updateTripDirectionId(GDirectionId.NEW_1.id, headSignToGTripIdInts[sortedHeadSigns[0]]) + routeGTFS.updateTripDirectionId(GDirectionId.NEW_2.id, headSignToGTripIdInts[sortedHeadSigns[1]]) MTLog.log("$routeId: Splitting directions... DONE (with trip head-signs)") return } @@ -84,7 +85,7 @@ object MDirectionSplitter { ) { MTLog.log("$routeId: Keep original direction ID") } else { - routeGTFS.updateTripDirectionId(GDirectionId.NEW_1, directionsCandidates[0].tripIdInts) + routeGTFS.updateTripDirectionId(GDirectionId.NEW_1.id, directionsCandidates[0].tripIdInts) } } @@ -95,8 +96,8 @@ object MDirectionSplitter { } else { GDirectionId.NEW_1 to GDirectionId.NEW_2 } - routeGTFS.updateTripDirectionId(direction0, directionsCandidates[0].tripIdInts) - routeGTFS.updateTripDirectionId(direction1, directionsCandidates[1].tripIdInts) + routeGTFS.updateTripDirectionId(direction0.id, directionsCandidates[0].tripIdInts) + routeGTFS.updateTripDirectionId(direction1.id, directionsCandidates[1].tripIdInts) } else -> {