diff --git a/app/build.gradle.kts b/app/build.gradle.kts index d6f274d..a6bb535 100644 --- a/app/build.gradle.kts +++ b/app/build.gradle.kts @@ -53,6 +53,7 @@ android { } testOptions { unitTests.isReturnDefaultValues = true + unitTests.all { it.maxHeapSize = "16m" } } compileOptions { sourceCompatibility = JavaVersion.VERSION_11 diff --git a/app/src/main/java/agzam/vyatsu/data/remote/GroupsParser.kt b/app/src/main/java/agzam/vyatsu/data/remote/GroupsParser.kt index 01a2be3..10dae46 100644 --- a/app/src/main/java/agzam/vyatsu/data/remote/GroupsParser.kt +++ b/app/src/main/java/agzam/vyatsu/data/remote/GroupsParser.kt @@ -3,8 +3,7 @@ package agzam.vyatsu.data.remote import agzam.vyatsu.domain.model.DateRange import agzam.vyatsu.domain.model.Group import agzam.vyatsu.domain.model.Season -import android.util.Log -import org.jsoup.Jsoup +import java.io.InputStream /** * Основано на https://github.com/alirzaev/vyatsu-schedule-backend/blob/master/utils/extractors.py @@ -14,6 +13,13 @@ object GroupsParser { private val groupPattern = Regex("""/reports/schedule/Group/(\d{4,})_([12])_(\d{8})_(\d{8})\.pdf""") + private val grpIdPattern = Regex("""data-grp_period_id="(\d+)"""") + private val parenthesized = Regex("""\(.*?\)\s?""") + + private const val CHUNK_SIZE = 8192 + private const val MAX_SLICE = 1 shl 16 // 64 KiB — защита от аномально длинных единиц + private const val MAX_PENDING = 1 shl 20 // 1 MiB — жёсткий потолок буфера + private val shorthands = mapOf( "Институт биологии и биотехнологии (факультет)(ОРУ)" to "ИББТ", "Институт химии и экологии (факультет) (ОРУ)" to "ИнХимЭк", @@ -33,40 +39,118 @@ object GroupsParser { ) /** - * Извлекает список групп с HTML страницы + * Стриминговый парсер страницы групп: читает тело ответа по чанкам и + * разбирает его на лету, не материализуя весь HTML в память. + * + * Страница весит ~2.6MB, а на устройствах с маленькой кучей даже + * одиночный String такого размера — критично (jsoup-DOM и подавно приводил к OOM). + * За один проход извлекаем и список групп, и диапазоны дат из PDF-ссылок. */ - fun parseGroups(html: String): List { - val doc = Jsoup.parse(html) - val container = doc.selectFirst("div.column-center_rasp") ?: return emptyList() - val tables = container.select("table[style~=border: none !important]") + fun parseStream(input: InputStream): ParsedGroups { + val reader = input.bufferedReader(Charsets.UTF_8) + val buf = CharArray(CHUNK_SIZE) + val pending = StringBuilder(CHUNK_SIZE * 2) + val groups = ArrayList() + val ranges = HashMap>>() + var faculty = "" - return tables.flatMap { table -> - table.select("> tbody > tr, > tr").flatMap { row -> - val faculty = shortname(row.selectFirst("div.fak_name")?.text().orEmpty()) // - - row.select("div.grpPeriod").map { grp -> - Group( - id = grp.attr("data-grp_period_id").dropLast(1), - name = removeParenthesized(grp.text()), - faculty = faculty - ) - } + fun addRanges(text: String) { + for (m in groupPattern.findAll(text)) { + val groupId = m.groupValues[1] + val season = if (m.groupValues[2] == "1") Season.AUTUMN else Season.SPRING + val range = DateRange(m.groupValues[3], m.groupValues[4]) + ranges.getOrPut(groupId) { + hashMapOf(Season.AUTUMN to ArrayList(), Season.SPRING to ArrayList()) + }.getOrPut(season) { ArrayList() }.add(range) } } + + while (true) { + val n = reader.read(buf) + if (n < 0) break + pending.append(buf, 0, n) + + var pos = 0 + while (pos < pending.length) { + val divAt = pending.indexOf(" aAt + aAt < 0 -> divAt + else -> minOf(divAt, aAt) + } + if (tagStart < 0) { + // хвостовой текст без div-тегов — обычный текст страницы + pos = pending.length + break + } + if (tagStart > pos) { + addRanges(pending.substring(pos, tagStart)) + } + val tagEnd = findTagEnd(pending, tagStart) + if (tagEnd < 0) break // тег не дочитан — ждём следующий чанк + val tagText = pending.substring(tagStart, tagEnd + 1) + + val isDiv = tagText.startsWith("", tagEnd + 1) + if (closeDiv < 0) break // содержимое не дочитано — ждём следующий чанк + addRanges(tagText) + val content = pending.substring(tagEnd + 1, closeDiv) + if (isFakName) { + val name = content.trim() + if (name.isNotEmpty()) faculty = shortname(name) + } else { + val m = grpIdPattern.find(tagText) + if (m != null) { + groups += Group( + id = m.groupValues[1].dropLast(1), + name = removeParenthesized(content), + faculty = faculty + ) + } + } + pos = closeDiv + 6 + } else { + addRanges(tagText) + pos = tagEnd + 1 + } + + if (pos > MAX_SLICE) { + pending.delete(0, pos) + pos = 0 + } + } + if (pos > 0 && pos <= pending.length) { + pending.delete(0, pos) + } + if (pending.length > MAX_PENDING) { + // на странице нет настолько длинных конструкций; страховка от аномалий + pending.delete(0, pending.length - MAX_SLICE) + } + } + return ParsedGroups(groups, ranges) } - fun parseDateRanges(html: String): Map>> { - val result = HashMap>>() - groupPattern.findAll(html).forEach { m -> - val groupId = m.groupValues[1] - val season = if (m.groupValues[2] == "1") Season.AUTUMN else Season.SPRING - val range = DateRange(m.groupValues[3], m.groupValues[4]) - result - .getOrPut(groupId) { hashMapOf(Season.AUTUMN to mutableListOf(), Season.SPRING to mutableListOf()) } - .getOrPut(season) { mutableListOf() } - .add(range) + /** Ищет конец HTML-тега — символ '>' вне кавычек. */ + private fun findTagEnd(s: CharSequence, start: Int): Int { + var i = start + var quote = 0.toChar() + while (i < s.length) { + val c = s[i] + if (quote != 0.toChar()) { + if (c == quote) quote = 0.toChar() + } else { + when (c) { + '"', '\'' -> quote = c + '>' -> return i + } + } + i++ } - return result + return -1 } private fun shortname(name: String): String { @@ -75,5 +159,10 @@ object GroupsParser { return if (sh != null) "$sh ($stripped)" else stripped } - private fun removeParenthesized(name: String): String = name.replace(Regex("""\(.*?\)\s?"""), "").trim() + private fun removeParenthesized(name: String): String = name.replace(parenthesized, "").trim() } + +data class ParsedGroups( + val groups: List, + val dateRanges: Map>> +) \ No newline at end of file diff --git a/app/src/main/java/agzam/vyatsu/data/remote/VyatsuHttpClient.kt b/app/src/main/java/agzam/vyatsu/data/remote/VyatsuHttpClient.kt index dc4e890..c71d7c6 100644 --- a/app/src/main/java/agzam/vyatsu/data/remote/VyatsuHttpClient.kt +++ b/app/src/main/java/agzam/vyatsu/data/remote/VyatsuHttpClient.kt @@ -5,6 +5,7 @@ import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.withContext import okhttp3.OkHttpClient import okhttp3.Request +import java.io.InputStream import java.util.concurrent.TimeUnit /** @@ -35,4 +36,15 @@ object VyatsuHttpClient { } } + /** + * Открывает потоковое тело ответа. Вызывающий обязан закрыть + * возвращённый [InputStream], чтобы вернуть соединение в пул. + */ + suspend fun openStream(url: String): InputStream = withContext(Dispatchers.IO) { + val resp = client.newCall(Request.Builder().url(url).build()).execute() + Log.d(TAG, "GET $url -> ${resp.code}") + check(resp.isSuccessful) { "HTTP ${resp.code} for $url" } + resp.body!!.byteStream() + } + } diff --git a/app/src/main/java/agzam/vyatsu/data/repository/GroupRepositoryImpl.kt b/app/src/main/java/agzam/vyatsu/data/repository/GroupRepositoryImpl.kt index e588d78..399b778 100644 --- a/app/src/main/java/agzam/vyatsu/data/repository/GroupRepositoryImpl.kt +++ b/app/src/main/java/agzam/vyatsu/data/repository/GroupRepositoryImpl.kt @@ -9,9 +9,11 @@ import agzam.vyatsu.domain.ScheduleDates import agzam.vyatsu.domain.model.Group import agzam.vyatsu.domain.model.Season import agzam.vyatsu.domain.repository.GroupRepository +import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.flow.Flow import kotlinx.coroutines.flow.first import kotlinx.coroutines.flow.map +import kotlinx.coroutines.withContext class GroupRepositoryImpl( private val groupDao: GroupDao, @@ -40,11 +42,11 @@ class GroupRepositoryImpl( } private suspend fun refresh() { - val html = VyatsuHttpClient.getText(VyatsuUrls.GROUPS_PAGE) - val groups = GroupsParser.parseGroups(html) - val ranges = GroupsParser.parseDateRanges(html) - val entities = groups.map { group -> - val range = ranges[group.id] + val parsed = withContext(Dispatchers.IO) { + VyatsuHttpClient.openStream(VyatsuUrls.GROUPS_PAGE).use(GroupsParser::parseStream) + } + val entities = parsed.groups.map { group -> + val range = parsed.dateRanges[group.id] val autumn = range?.get(Season.AUTUMN)?.lastOrNull() val spring = range?.get(Season.SPRING)?.lastOrNull() GroupEntity(