diff --git a/build.sbt b/build.sbt index 90787c18..a97b9989 100644 --- a/build.sbt +++ b/build.sbt @@ -143,7 +143,8 @@ lazy val ce3 = (project in file("ce3")) Libraries.refinedCore, Libraries.shapeless, "com.kubukoz" %% "debug-utils" % "1.1.3", - Libraries.catsMtl + Libraries.catsMtl, + Libraries.jsoup, ) ) .enablePlugins(ScalaxbPlugin) diff --git a/ce3/src/main/scala/gas104/Credentials.scala b/ce3/src/main/scala/gas104/Credentials.scala new file mode 100644 index 00000000..21bf932d --- /dev/null +++ b/ce3/src/main/scala/gas104/Credentials.scala @@ -0,0 +1,11 @@ +package gas104 + +object Credentials { + + val login104 = "" + val password104 = "" + val sessionId104 = "" + val account104 = "" + val meter104 = "" + +} diff --git a/ce3/src/main/scala/gas104/DomainSpec.scala b/ce3/src/main/scala/gas104/DomainSpec.scala index 2593812d..2bf4e65a 100644 --- a/ce3/src/main/scala/gas104/DomainSpec.scala +++ b/ce3/src/main/scala/gas104/DomainSpec.scala @@ -1,26 +1,28 @@ package gas104 -import cats.effect.Concurrent import cats.effect.IO -import cats.effect.Sync -import cats.effect.kernel.Async -import cats.effect.kernel.Resource import cats.implicits._ +import gas104.Htttp._ import gas104.domain._ import gas104.domain.api._ import io.circe.parser import io.circe.syntax.EncoderOps + import java.time.Instant import java.time.LocalDateTime -import org.http4s.Response -import org.http4s.blaze.client.BlazeClientBuilder -import org.http4s.client.Client -import org.scalatest.Succeeded +import org.http4s.Header +import org.http4s.Method +import org.http4s.Request +import org.http4s.client.middleware.FollowRedirect +import org.http4s.implicits.http4sLiteralsSyntax import org.scalatest.funspec.AnyFunSpec import org.scalatest.matchers.should.Matchers +import org.typelevel.ci.CIStringSyntax class DomainSpec extends AnyFunSpec with Matchers { + import Credentials._ + describe("Row object") { val rowObj: Row = Row( @@ -67,7 +69,7 @@ class DomainSpec extends AnyFunSpec with Matchers { 0, 1689886800, ) - ) + ).some ) val rawData = @@ -133,41 +135,180 @@ class DomainSpec extends AnyFunSpec with Matchers { } - describe("http") { + describe("read meters - token required") { + + it("1") { - def body[F[_] : Concurrent](rs: Response[F]): F[String] = - rs.body - .through(fs2.text.utf8.decode[F]) - .compile - .foldMonoid - def mkHttpClient[F[_]: Async]: Resource[F, Client[F]] = - BlazeClientBuilder[F].resource + import cats.effect.unsafe.implicits.global - def mkRequest[F[_]: Concurrent](client: Client[F]) = { - import org.http4s.circe.CirceEntityCodec.circeEntityDecoder - client.expect[Data](Htttp.rq[F]) + mkHttpClient[IO] + .use(obtainData[IO](sessionId104)) + .flatMap(representData[IO]) + .unsafeRunSync() } - def representData[F[_]: Sync](payload: Data): F[Unit] = payload.data - .traverse_ { x: Row => - val r = URow.from(x) - val line = (r.dateTime, r.counter, r.delta) - Sync[F].delay(pprint.pprintln(line)) - } + } + + describe("getting csrf token") { it("1") { import cats.effect.unsafe.implicits.global - mkHttpClient[IO] - .use(mkRequest[IO]) - .flatMap(representData[IO]) + + val x = obtainCsrfToken[IO] .unsafeRunSync() + pprint.pprintln(x) } - it("stub tun trigger compilation") { - Succeeded + } + + describe("try to obtain session id") { + it("1") { + import cats.effect.unsafe.implicits.global + + //////////////// CSRF TOKEN //////////////// + val (csrf, (ck, cv)) = obtainCsrfToken[IO].unsafeRunSync().get + pprint.pprintln(csrf) + pprint.pprintln(ck) + pprint.pprintln(cv) + + val rq: Request[IO] = mkRequestLogin[IO](login104, password104, csrf, (ck, cv)) + + //////////////// LOGIN //////////////// + val (body0, hs, st) = mkHttpClient[IO] + .flatMap(_.run(rq)) + .use { rs => + body(rs).map((_, rs.headers, rs.status)) + } + .unsafeRunSync() + + val (ck2, cv2) = extractCookie(hs).get +// .foreach(h => pprint.pprintln(h)) + + pprint.pprintln(st) + pprint.pprintln(body0) + + val url = uri"https://account.104.ua/ua/account/index" + + val index = Request[IO](Method.GET, url) + .withHeaders(Header.Raw(ci"$ck2", cv2)) + + //////////////// /ua/account/index //////////////// + val (b, s, hs2) = mkHttpClient[IO] + .flatMap(_.run(index)) + .use { rs => + body(rs) + .map(b => + ( + b, + rs.status, + rs.headers + ) + ) + }.unsafeRunSync() + + pprint.pprintln(b) + pprint.pprintln(s) + hs2.foreach(x => pprint.pprintln(x)) + + //////////////// /ua/login //////////////// + val (ck3, cv3) = extractCookie(hs2).get + val urlLogin = uri"https://account.104.ua/ua/login" + val login = Request[IO](Method.GET, urlLogin) + .withHeaders(Header.Raw(ci"$ck3", cv3)) + + val (b2, s2, hs3) = mkHttpClient[IO] + .flatMap(_.run(login)) + .use { rs => + body(rs) + .map(b => + ( + b, + rs.status, + rs.headers + ) + ) + }.unsafeRunSync() + + pprint.pprintln(b2) + pprint.pprintln(s2) + hs3.foreach(x => pprint.pprintln(x)) + } + it("2") { + import cats.effect.unsafe.implicits.global + + //////////////// CSRF TOKEN //////////////// + val (csrf, (ck, cv)) = obtainCsrfToken[IO].unsafeRunSync().get + pprint.pprintln(csrf) + pprint.pprintln(ck) + pprint.pprintln(cv) + + val rq: Request[IO] = mkRequestLogin[IO](login104, password104, csrf, (ck, cv)) + //////////////// LOGIN //////////////// + val (body0, hs, st) = mkHttpClient[IO] + .map(cl => FollowRedirect(5, _ => true)(cl)) + .flatMap(_.run(rq)) + .use { rs => + body(rs).map((_, rs.headers, rs.status)) + } + .unsafeRunSync() + + val (ck2, cv2) = extractCookie(hs).get +// .foreach(h => pprint.pprintln(h)) + + hs.headers.foreach(x => pprint.pprintln(x)) + pprint.pprintln(st) + pprint.pprintln(body0) +// +// val url = uri"https://account.104.ua/ua/account/index" +// +// val index = Request[IO](Method.GET, url) +// .withHeaders(Header.Raw(ci"$ck2", cv2)) +// +// //////////////// /ua/account/index //////////////// +// val (b, s, hs2) = mkHttpClient[IO] +// .flatMap(_.run(index)) +// .use { rs => +// body(rs) +// .map(b => +// ( +// b, +// rs.status, +// rs.headers +// ) +// ) +// }.unsafeRunSync() +// +// pprint.pprintln(b) +// pprint.pprintln(s) +// hs2.foreach(x => pprint.pprintln(x)) +// +// //////////////// /ua/login //////////////// +// val (ck3, cv3) = extractCookie(hs2).get +// val urlLogin = uri"https://account.104.ua/ua/login" +// val login = Request[IO](Method.GET, urlLogin) +// .withHeaders(Header.Raw(ci"$ck3", cv3)) +// +// val (b2, s2, hs3) = mkHttpClient[IO] +// .flatMap(_.run(login)) +// .use { rs => +// body(rs) +// .map(b => +// ( +// b, +// rs.status, +// rs.headers +// ) +// ) +// }.unsafeRunSync() +// +// pprint.pprintln(b2) +// pprint.pprintln(s2) +// hs3.foreach(x => pprint.pprintln(x)) +// + } } } diff --git a/ce3/src/main/scala/gas104/GasApp.scala b/ce3/src/main/scala/gas104/GasApp.scala deleted file mode 100644 index e2fd3ca3..00000000 --- a/ce3/src/main/scala/gas104/GasApp.scala +++ /dev/null @@ -1,20 +0,0 @@ -package gas104 - -import cats.effect.IO -import cats.effect.IOApp -import io.circe.generic.AutoDerivation -import io.circe.generic.extras.Configuration -import io.circe.generic.extras.semiauto.{deriveConfiguredDecoder, deriveConfiguredEncoder} -import io.circe.syntax.EncoderOps -import io.circe.{Decoder, Encoder} - -object GasApp extends IOApp.Simple { - - - override def run: IO[Unit] = IO{ - - println( - - ) - } -} diff --git a/ce3/src/main/scala/gas104/Htttp.scala b/ce3/src/main/scala/gas104/Htttp.scala index 8c9ba307..05e57920 100644 --- a/ce3/src/main/scala/gas104/Htttp.scala +++ b/ce3/src/main/scala/gas104/Htttp.scala @@ -1,36 +1,138 @@ package gas104 +import cats.effect._ +import cats.implicits._ +import gas104.Credentials.{account104, meter104} +import gas104.domain.URow +import gas104.domain.api.Data +import gas104.domain.api.Row import org.http4s.Headers import org.http4s.MediaType import org.http4s.Method import org.http4s.Request import org.http4s.RequestCookie +import org.http4s.Response import org.http4s.UrlForm +import org.http4s.blaze.client.BlazeClientBuilder +import org.http4s.client.Client import org.http4s.headers._ import org.http4s.implicits.http4sLiteralsSyntax +import org.typelevel.ci.CIStringSyntax object Htttp { + /** convenient header builder */ object PhpSessionId { def apply(value: String): RequestCookie = RequestCookie("PHPSESSID", value) } - val uri = uri"https://ok.104.ua/ua/ajx/individual/meterage/history/remote" - val headers = Headers( - `Accept`(MediaType.application.json), - `Content-Type`(MediaType.application.`x-www-form-urlencoded`), - `Cookie`(PhpSessionId("")), - ) - - val payload = UrlForm( - "account_no" -> "", // 0800xxxxxx - "meter_no" -> "", - "period_type" -> "y", - "end_date" -> "2023-12-31T23:59:59.999Z" - ) - - def rq[F[_]] = Request[F](Method.POST, uri) - .withHeaders(headers) - .withEntity(payload) + /** Response[F] => F[String] */ + def body[F[_]: Concurrent](rs: Response[F]): F[String] = + rs.body + .through(fs2.text.utf8.decode[F]) + .compile + .foldMonoid + + def extractCookie(hs: Headers) = + hs.headers + .find(h => h.name.contains(ci"cookie")) + .map(_.value) + .map(_.split(";").apply(0)) + .map(_.split("=")) + .flatMap { + case Array(k, v) => (k -> v).some + case _ => None + } + + /** Response[F] => F[Option[String]] */ + def extractToken[F[_]: Concurrent](rs: Response[F]) = + rs.body + .through(fs2.text.utf8.decode[F]) + .through(fs2.text.lines[F]) + .dropWhile(s => !s.contains("csrf_token")) + .drop(1) + .take(1) + .map(_.split("=")) + .filter(_.length == 2) + .map(_(1).stripPrefix("\"").stripSuffix("\"")) + .compile + .last + .map(maybeToken => (maybeToken, extractCookie(rs.headers)).mapN(_ -> _)) + + /** => Option[String] */ + def obtainCsrfToken[F[_]: Async] = { + + import org.http4s.Method + + val uri = uri"https://account.104.ua/ua/login" + val rq = Request[F]( + Method.GET, + uri + ) + mkHttpClient[F] + .flatMap(_.run(rq)) + .use(extractToken[F]) + } + + /** => Client[F] */ + def mkHttpClient[F[_]: Async]: Resource[F, Client[F]] = + BlazeClientBuilder[F].resource + + /** login: */ + def mkRequestLogin[F[_]](login: String, password: String, csrfToken: String, cookie: (String, String)): Request[F] = { + val url = uri"https://account.104.ua/ua/login" + val headers = Headers( + `Content-Type`(MediaType.application.`x-www-form-urlencoded`), + Cookie(RequestCookie(cookie._1, cookie._2)), + ) + val form = UrlForm( + "username" -> login, + "password" -> password, + "_csrf_token" -> csrfToken, + ) + Request[F](Method.POST, url) + .withHeaders(headers) + .withEntity(form) + } + + /** meters: sessionId => Request[F] */ + def mkRequestData[F[_]](phpSessionId: String): Request[F] = { + + val url = uri"https://ok.104.ua/ua/ajx/individual/meterage/history/remote" + + def mkHeaders(value: String) = Headers( + `Accept`(MediaType.application.json), + `Content-Type`(MediaType.application.`x-www-form-urlencoded`), + `Cookie`(PhpSessionId(value)), + ) + + val payloadMeters = UrlForm( + "account_no" -> account104, + "meter_no" -> meter104, + "period_type" -> "y", +// "end_date" -> "2023-12-31T23:59:59.999Z" + ) + + Request[F](Method.POST, url) + .withHeaders(mkHeaders(phpSessionId)) + .withEntity(payloadMeters) + } + + def obtainData[F[_]: Concurrent](sessionId: String)(client: Client[F]): F[Data] = { + import org.http4s.circe.CirceEntityCodec.circeEntityDecoder + client.expect[Data](mkRequestData[F](sessionId)) + } + + def representData[F[_]: Sync](payload: Data): F[Unit] = + payload.data match { + case Some(data) => + data.traverse_ { x: Row => + val r = URow.from(x) + val line = (r.dateTime, r.counter, r.delta) + Sync[F].delay(pprint.pprintln(line)) + } + case None => + Sync[F].delay(pprint.pprintln("Error" -> payload)) + } } diff --git a/ce3/src/main/scala/gas104/domain.scala b/ce3/src/main/scala/gas104/domain.scala index 61bfb5b5..4d16cd15 100644 --- a/ce3/src/main/scala/gas104/domain.scala +++ b/ce3/src/main/scala/gas104/domain.scala @@ -29,7 +29,7 @@ object domain { implicit val decoder: Decoder[Row] = deriveConfiguredDecoder } - case class Data(error: Option[String], data: Seq[Row]) + case class Data(error: Option[String], data: Option[Seq[Row]]) object Data extends AutoDerivation diff --git a/ce3/src/main/scala/gas104/login.http b/ce3/src/main/scala/gas104/login.http new file mode 100644 index 00000000..a664c07a --- /dev/null +++ b/ce3/src/main/scala/gas104/login.http @@ -0,0 +1,10 @@ +#### +GET https://account.104.ua/ua/login + +#### +POST https://account.104.ua/ua/login +Content-Type: application/x-www-form-urlencoded + +username = & +password = & +_csrf_token= \ No newline at end of file