Merge pull request #1 from lichifeng/dev

merge de 75350 analyzing from aoc-mgz

CMakeLists.txt

@@ -3,9 +3,9 @@
 # Project info
 project(
         MgxParser
-        VERSION 0.4.1
+        VERSION 0.4.3
         LANGUAGES CXX
-        HOMEPAGE_URL "aocrec.com"
+        HOMEPAGE_URL "https://github.com/lichifeng/MgxParser"
         DESCRIPTION "MgxParser is a C++ lib used to parse Age of Empires II game records."
 )
 

README.md

@@ -1,5 +1,5 @@
 # **MgxParser**
-*This version(0.4.1) was compiled on 20230129*
+*This version(0.4.3) was compiled on 20230221*
 
 ## Introduction
 MgxParser is a C++ lib used to parse Age of Empires II game records.

src/analyzers/default/mainproc_extractstreams.cc

@@ -6,6 +6,7 @@
  ***************************************************************/
 
 #include <array>
+#include <sstream>
 
 #include "analyzer.h"
 #include "searcher.h"
@@ -30,12 +31,33 @@ bool DefaultAnalyzer::ExtractStreams() {
     uint32_t *compressed_size_p;
     uint16_t *namelen_p;
     uint16_t *exlen_p;
+    uint16_t *raw_msdos_time_p;
+    uint16_t *raw_msdos_date_p;
 
     if (is_zip) {
         compressed_size_p = (uint32_t *)(input_start + 18);
         namelen_p = (uint16_t *)(input_start + 26);
         exlen_p = (uint16_t *)(input_start + 28);
 
+        // get modified time of file, https://groups.google.com/g/comp.os.msdos.programmer/c/ffAVUFN2NbA
+        raw_msdos_time_p = (uint16_t *)(input_start + 10);
+        raw_msdos_date_p = (uint16_t *)(input_start + 12);
+        int year, mon, day, hour, minute, sec;
+        year = (int)(((*raw_msdos_date_p) >> 9) & 0x7f) + 1980;
+        mon = ((*raw_msdos_date_p) >> 5) & 0x0f;
+        day = (*raw_msdos_date_p) & 0x1f;
+        hour = (*raw_msdos_time_p) >> 11;
+        minute = ((*raw_msdos_time_p) >> 5) & 0x3f;
+        sec = (*raw_msdos_time_p) & 0x1f * 2;
+        std::stringstream iso_time_str;
+        iso_time_str << std::setfill ('0') << std::setw(4) << year << '-' 
+                     << std::setw(2) << mon << '-' 
+                     << std::setw(2) << day << 'T' 
+                     << std::setw(2) << hour << ':' 
+                     << std::setw(2) << minute << ':' 
+                     << std::setw(2) << sec << ".000Z";
+        modified_date_ = iso_time_str.str();
+
         std::vector<RECBYTE> outbuffer;
         if (0 != ZipDecompress(const_cast<uint8_t *>(input_start + 30 + *namelen_p + *exlen_p),
                                input_size_,  // zip64 have compressed_size == 0, so use input_size instead.

src/analyzers/default/mainproc_jsondump.cc

@@ -53,6 +53,8 @@ std::string DefaultAnalyzer::JsonOutput() {
     j["status"] = status_.body_scanned_ ? "perfect" : status_.mapdata_found_ ? "good" : status_.stream_extracted_ ? "valid" : "invalid";
     j["filetype"] = input_ext_;
     j["filename"] = input_filename_;
+    if (!modified_date_.empty())
+        j["modifiedDate"] = modified_date_;
     if (!file_md5_.empty())
         j["filemd5"] = file_md5_;
     if (!extracted_file_.empty())
@@ -189,7 +191,7 @@ std::string DefaultAnalyzer::JsonOutput() {
             pj["imperialTime"] = p.imperial_time_;
         pj["disconnected"] = p.disconnected_;
         pj["isWinner"] = p.is_winner_;
-        pj["colorIndex"] = p.color_id_;
+        pj["colorIndex"] = UINT32_INIT == p.dd_color_id_ ? p.dd_color_id_ : p.color_id_;
 
         j["players"].emplace_back(std::move(pj));
     }

src/analyzers/default/subproc_deheader.cc

@@ -8,6 +8,15 @@
 #include "analyzer.h"
 #include "searcher.h"
 
+void SkipStringBlock(RecCursor& cursor) {
+    uint32_t crc;
+    cursor >> crc;
+    if (0 == crc || crc > 255) {
+        cursor.ScanString();
+        SkipStringBlock(cursor);
+    }
+}
+
 void DefaultAnalyzer::AnalyzeDEHeader(int debugFlag) {
     if (!IS_DE(version_code_))
         return;
@@ -70,7 +79,8 @@ void DefaultAnalyzer::AnalyzeDEHeader(int debugFlag) {
     }
 
     // Read player data
-    for (size_t i = 1; i < 9; i++) {
+    uint32_t total_players = save_version_ > 36.9999 ? dd_numplayers_ + 1 : 9; // actually 8 players
+    for (size_t i = 1; i < total_players; i++) {
         cursor_ >> players[i].dd_dlc_id_
                 >> players[i].dd_color_id_
                 >> players[i].de_selected_color_
@@ -101,8 +111,15 @@ void DefaultAnalyzer::AnalyzeDEHeader(int debugFlag) {
     cursor_ >> 9
             >> 1 //_readBytes(1, &DE_fogOfWar);
             >> dd_cheat_notifications_
-            >> dd_colored_chat_
-            >> 4 // 0xa3, 0x5f, 0x02, 0x00
+            >> dd_colored_chat_;
+    if (save_version_ > 36.9999) {
+        cursor_ >> 4 * 3;
+        cursor_.ScanString();
+        cursor_ >> 1;
+        cursor_.ScanString().ScanString();
+        cursor_ >> (22 + 4 * 2 + 8);
+    }
+    cursor_ >> 4 // 0xa3, 0x5f, 0x02, 0x00
             >> dd_is_ranked_
             >> dd_allow_specs_
             >> dd_lobby_visibility_
@@ -111,46 +128,15 @@ void DefaultAnalyzer::AnalyzeDEHeader(int debugFlag) {
     if (save_version_ >= 13.1299)
         cursor_ >> de_spec_dely_ >> de_scenario_civ_;
 
-    // if (saveVersion >= 13.1299)
-    //     DE_RMSCrc = BytesToHex(_curPos, 4, true);
-
-    /// \warning 实话我也不知道这一段是什么鬼东西，只好用搜索
-    /// \todo aoc-mgz有更新，可以参考
-    /// https://github.com/happyleavesaoc/aoc-mgz/commit/4ffe9ad918b888531fc2e94c2b184cbd04ca9fb5
-    /// \note
-    /// de-13.03.aoe2record : 2a 00 00 00 fe ff ff ff + 59*(fe ff ff ff)
-    /// de-13.06.aoe2record : 2A 00 00 00 FE FF FF FF + 59*(00 00 00 00)
-    /// de-13.07.aoe2record : 2A 00 00 00 FE FF FF FF + 59*(fe ff ff ff)
-    /// de-13.08.aoe2record : 2A 00 00 00 FE FF FF FF + 59*(fe ff ff ff)
-    /// de-13.13.aoe2record : 2A 00 00 00 FE FF FF FF + 59*(00 00 00 00)
-    /// de-13.15.aoe2record : 2A 00 00 00 FE FF FF FF + 59*(00 00 00 00)
-    /// de-13.17.aoe2record : 2C 00 00 00 FE FF FF FF + 59*(00 00 00 00)
-    /// de-13.20.aoe2record : 2C 00 00 00 FE FF FF FF + 59*(00 00 00 00)
-    /// de-13.34.aoe2record : 2D 00 00 00 FE FF FF FF + 59*(00 00 00 00)
-    /// de-20.06.aoe2record : 2D 00 00 00 FE FF FF FF + 59*(00 00 00 00)
-    /// de-20.16.aoe2record : 2D 00 00 00 FE FF FF FF + 59*(00 00 00 00)
-    /// de-25.01.aoe2record : 2E 00 00 00 FE FF FF FF + 59*(00 00 00 00)
-    /// de-25.02.aoe2record : 2E 00 00 00 FE FF FF FF + 59*(00 00 00 00)
-    /// de-25.06.aoe2record : 2F 00 00 00 FE FF FF FF + 59*(00 00 00 00)
-    /// de-25.22.aoe2record : 2F 00 00 00 00 00 00 00 + NOTHING
-    /// de-26.16.aoe2record : 2F 00 00 00 00 00 00 00 + NOTHING
-    /// de-26.18.aoe2record : 2F 00 00 00 00 00 00 00 + NOTHING
-    /// de-26.21.aoe2record : 2F 00 00 00 00 00 00 00 + NOTHING
-    std::array<uint8_t, 2> hdstring_separator = {0x60, 0x0a};
-    auto found = cursor_.Itr();
-    auto header_end = cursor_.Itr(body_start_);
-    for (size_t i = 0; i < 23; i++) {
-        found = SearchPattern(
-                found, header_end,
-                hdstring_separator.begin(),
-                hdstring_separator.end());
-        if (found != header_end)
-            found += (4 + *((uint16_t *) (&(*found) + 2)));
+    // https://github.com/happyleavesaoc/aoc-mgz/blob/dd4d122259e3b97fe09ac08aa75fe7ff5a75c72d/mgz/header/de.py#L130
+    SkipStringBlock(cursor_);
+    cursor_ >> 8;
+    for (size_t i = 0; i < 20; i++) {
+        SkipStringBlock(cursor_);
     }
-    cursor_(found) >> 4; // 2a/c/d/e/f 00 00 00
     if (save_version_ >= 25.2199) {
-        int tmp_len;
-        cursor_ >> tmp_len >> 4 * tmp_len;
+        int num_sn;
+        cursor_ >> num_sn >> 4 * num_sn;
     } else {
         cursor_ >> 60 * 4;
     }
@@ -181,13 +167,17 @@ void DefaultAnalyzer::AnalyzeDEHeader(int debugFlag) {
         cursor_ >> 4;
     if (save_version_ >= 26.1599)
         cursor_ >> 8;
+    if (save_version_ >= 36.9999)
+        cursor_ >> 3;
     cursor_.ScanString() >> 5;
     if (save_version_ >= 13.1299)
         cursor_ >> 1;
     if (save_version_ < 13.1699)
         cursor_.ScanString() >> 8; // uint32 + 00 00 00 00
     if (save_version_ >= 13.1699)
         cursor_ >> 2;
+    if (save_version_ >= 36.9999)
+        cursor_ >> 8;
 
     ai_start_ = cursor_();
 }

src/analyzers/default/subproc_detectversion.cc

@@ -11,6 +11,12 @@ void DefaultAnalyzer::DetectVersion() {
     if (version_code_ != AOK)
         cursor_(body_start_) >> log_version_;
     cursor_(header_start_) >> version_string_ >> save_version_;
+    // From DE version 75350
+    if (save_version_ == -1) {
+        uint32_t new_saveversion;
+        cursor_ >> new_saveversion;
+        save_version_ = (float)new_saveversion;
+    }
     version_end_ = ai_start_ = cursor_();
 
     /// \todo Every condition needs to be tested!

src/analyzers/default/subproc_lobby.cc

@@ -17,6 +17,8 @@ void DefaultAnalyzer::AnalyzeLobby(int debug_flag) {
         cursor_ >> 9;
     if (save_version_ >= 26.1599)
         cursor_ >> 5;
+    if (save_version_ >= 36.9999)
+        cursor_ >> 8;
     for (size_t i = 1; i < 9; i++) {
         if (players[i].resolved_teamid_ != 255) {
             cursor_ >> 1;

src/analyzers/default/subproc_map.cc

@@ -51,13 +51,7 @@ void DefaultAnalyzer::AnalyzeMap(int debug_flag) {
     cursor_ >> restore_time_;
 
     uint32_t num_particles;
-    cursor_ >> num_particles >> (27 * num_particles);
-
-    // A checkpoint, expecting 10060 with AOK and 40600 with higher version
-    // borrow local var check_val
-    cursor_ >> check_val;
-    if (40600 != check_val && 10060 != check_val) // 10060 in AOK
-        throw std::string("Cannot find expected check value 10060/40600 in init Info section.");
+    cursor_ >> num_particles >> (27 * num_particles) >> 4;
 
     initinfo_start_ = cursor_();
     status_.mapdata_found_ = true;

src/datamodels/record.h

@@ -33,6 +33,7 @@ class Record {
     std::string input_ext_; ///< .zip/.mgx/.aoe2record/.mgz/.mgx2, etc.
     std::string extracted_file_; ///< filename of extracted record from .zip archive
     std::string file_md5_;
+    std::string modified_date_; // only for unzipped files
 
     // Version-related members
     uint32_t log_version_ = UINT32_INIT;     ///< body 的前4个字节，与版本有关，可以识别A/C版

test/test.cc

@@ -236,6 +236,18 @@ TEST_F(ParserTest, DE66692)
     EXPECT_EQ(recA["gameTime"], 1665150274);
 }
 
+// Test: de build 75350
+TEST_F(ParserTest, DE75350)
+{
+    load(recA, "de-versions/AgeIIDE_Replay_212394317.zip");
+    EXPECT_EQ(recA["version"]["build"], 75350);
+    EXPECT_EQ(recA["version"]["code"], "DE");
+    EXPECT_EQ(recA["version"]["interVer"], 1000);
+    EXPECT_EQ(recA["status"], "perfect");
+    EXPECT_EQ(recA["guid"], "3cf58e536108d3459b8f62554e2159c9");
+    EXPECT_EQ(recA["gameTime"], 1676955878);
+}
+
 // Test: Brutal search for player data position in initial section
 TEST_F(ParserTest, InitialDataBrutalSearch)
 {