zhushenwudi/SifCard
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

爬虫写入数据库

Browse Source
This commit is contained in:
zhushenwudi
2025-01-14 15:23:59 +08:00
parent 33e34c680e
commit da848c9128
4 changed files with 131 additions and 59 deletions
Download Patch File Download Diff File Expand all files Collapse all files

138
src/main/kotlin/Main.kt
View File

@@ -1,3 +1,5 @@
import database.TCard
import kotlinx.coroutines.delay
import kotlinx.serialization.ExperimentalSerializationApi
import kotlinx.serialization.encodeToString
import kotlinx.serialization.json.Json
@@ -6,10 +8,12 @@ import model.Card
import model.GameCh
import model.parseType
import org.jsoup.Jsoup
import org.ktorm.database.Database
import org.ktorm.dsl.*
import java.io.File
@ExperimentalSerializationApi
fun main(args: Array<String>) {
suspend fun main(args: Array<String>) {
val cardList = readJson<List<Card>>(fileName = "card.json")
cardList.forEach { card ->
card.illustId = "c_${card.illustId}.png"
@@ -18,62 +22,100 @@ fun main(args: Array<String>) {
val gameList = readJson<List<GameCh>>(fileName = "gamerch.json")
val database = Database.connect(
url = "jdbc:mysql://localhost:3306/sif_card",
driver = "com.mysql.jdbc.Driver",
user = "root",
password = ""
)
cardList.forEach { card ->
gameList.find { gameCh -> gameCh.name.contains(card.name) }?.let { gameCh ->
println(gameCh.url)
val doc = Jsoup.connect(gameCh.url).get()
val elements = doc.select(".markup.mu")
val tables = elements.select(".mu__table")
tables.forEach { element ->
println(element)
println("------------")
database.useTransaction {
val count = database
.from(TCard)
.select()
.where { TCard.name eq card.name }
.totalRecordsInAllPages
if (count > 0) {
return@forEach
}
card.characterName = elements.select(".mu__index").select("p").text().substringBefore(")").substringAfter("］")
gameList.find { gameCh -> gameCh.name.contains(card.name) }?.let { gameCh ->
println(gameCh.url)
val doc = Jsoup.connect(gameCh.url).get()
val elements = doc.select(".markup.mu")
tables[1].select(".mu__table--row1").apply {
card.rarity = select(".mu__table--col2").text()
card.maxLevel = select(".mu__table--col4").text()
}
tables[1].select(".mu__table--row2").apply {
select(".mu__table--col2").text().let {
card.type = parseType(it)
val tables = elements.select(".mu__table")
card.characterName = elements.select(".mu__index").select("p").text()
.substringBefore(")").substringAfter("］")
tables[1].select(".mu__table--row1").apply {
card.rarity = select(".mu__table--col2").text()
card.maxLevel = select(".mu__table--col4").text()
}
tables[1].select(".mu__table--row2").apply {
select(".mu__table--col2").text().let {
card.type = parseType(it)
}
}
tables[1].select(".mu__table--row3").apply {
card.group = select(".mu__table--col2").text()
card.team = select(".mu__table--col4").text()
}
tables[1].select(".mu__table--row4").apply {
card.date = select(".mu__table--col2").text().substringBefore("（")
}
tables[2].select(".mu__table--row1").apply {
card.maxHp = select(".mu__table--col2").text()
card.smile = select(".mu__table--col4").text()
}
tables[2].select(".mu__table--row2").apply {
card.pure = select(".mu__table--col2").text()
card.cool = select(".mu__table--col4").text()
}
tables[3].select(".mu__table--row1").apply {
card.centerSkillName = select(".mu__table--col1").text()
}
tables[3].select(".mu__table--row2").apply {
card.centerSkillInfo = select(".mu__table--col1").text()
}
tables[4].select(".mu__table--row1").apply {
card.specialSkillName = select(".mu__table--col1").text()
}
tables[4].select(".mu__table--row2").apply {
card.specialSkillInfo = select(".mu__table--col1").text()
}
}
tables[1].select(".mu__table--row3").apply {
card.group = select(".mu__table--col2").text()
card.team = select(".mu__table--col4").text()
}
tables[1].select(".mu__table--row4").apply {
card.date = select(".mu__table--col2").text().substringBefore("（")
}
println(card)
tables[2].select(".mu__table--row1").apply {
card.maxHp = select(".mu__table--col2").text()
card.smile = select(".mu__table--col4").text()
}
tables[2].select(".mu__table--row2").apply {
card.pure = select(".mu__table--col2").text()
card.cool = select(".mu__table--col4").text()
}
tables[3].select(".mu__table--row1").apply {
card.centerSkillName = select(".mu__table--col1").text()
}
tables[3].select(".mu__table--row2").apply {
card.centerSkillInfo = select(".mu__table--col1").text()
}
tables[4].select(".mu__table--row1").apply {
card.specialSkillName = select(".mu__table--col1").text()
}
tables[4].select(".mu__table--row2").apply {
card.specialSkillInfo = select(".mu__table--col1").text()
database.insert(TCard) {
set(it.name, card.name)
set(it.characterId, card.characterId)
set(it.characterName, card.characterName)
set(it.type, card.type)
set(it.rarity, card.rarity)
set(it.maxLevel, card.maxLevel)
set(it.maxHp, card.maxHp)
set(it.smile, card.smile)
set(it.pure, card.pure)
set(it.cool, card.cool)
set(it.illustId, card.illustId)
set(it.evolveIllustId, card.evolveIllustId)
set(it.group, card.group)
set(it.team, card.team)
set(it.date, card.date)
set(it.centerSkillName, card.centerSkillName)
set(it.centerSkillInfo, card.centerSkillInfo)
set(it.specialSkillName, card.specialSkillName)
set(it.specialSkillInfo, card.specialSkillInfo)
}
}
println(card)
return
delay(1000)
}
}