Merge pull request #12165 from noahpistilli/scheduler-downloader

IOS/KD: Implement Download Scheduler

Source/Core/Core/IOS/Network/KD/NWC24Config.h

@@ -32,6 +32,7 @@ enum ErrorCode : s32
   WC24_ERR_ID_NONEXISTANCE = -34,
   WC24_ERR_ID_GENERATED = -35,
   WC24_ERR_ID_REGISTERED = -36,
+  WC24_ERR_DISABLED = -39,
   WC24_ERR_ID_NOT_REGISTERED = -44,
 };
 

Source/Core/Core/IOS/Network/KD/NWC24DL.cpp

@@ -59,7 +59,7 @@ void NWC24Dl::WriteDlList() const
     ERROR_LOG_FMT(IOS_WC24, "Failed to open or write WC24 DL list file");
 }
 
-bool NWC24Dl::DoesEntryExist(u16 entry_index)
+bool NWC24Dl::DoesEntryExist(u16 entry_index) const
 {
   return m_data.entries[entry_index].low_title_id != 0;
 }
@@ -125,6 +125,76 @@ bool NWC24Dl::IsRSASigned(u16 entry_index) const
   return !Common::ExtractBit(Common::swap32(m_data.entries[entry_index].flags), 2);
 }
 
+bool NWC24Dl::SkipSchedulerDownload(u16 entry_index) const
+{
+  // Some entries can be set to not be downloaded by the scheduler.
+  return !!Common::ExtractBit(Common::swap32(m_data.entries[entry_index].flags), 5);
+}
+
+bool NWC24Dl::HasSubtask(u16 entry_index) const
+{
+  switch (m_data.entries[entry_index].subtask_type)
+  {
+  case 1:
+  case 2:
+  case 3:
+  case 4:
+    return true;
+  default:
+    return false;
+  }
+}
+
+bool NWC24Dl::IsSubtaskDownloadDisabled(u16 entry_index) const
+{
+  return !!Common::ExtractBit(Common::swap16(m_data.entries[entry_index].subtask_flags), 9);
+}
+
+bool NWC24Dl::IsValidSubtask(u16 entry_index, u8 subtask_id) const
+{
+  return !!Common::ExtractBit(m_data.entries[entry_index].subtask_bitmask, subtask_id);
+}
+
+u64 NWC24Dl::GetNextDownloadTime(u16 record_index) const
+{
+  // Timestamps are stored as a UNIX timestamp but in minutes. We want seconds.
+  return Common::swap32(m_data.records[record_index].next_dl_timestamp) * SECONDS_PER_MINUTE;
+}
+
+u64 NWC24Dl::GetRetryTime(u16 entry_index) const
+{
+  const u64 retry_time = Common::swap16(m_data.entries[entry_index].retry_frequency);
+  if (retry_time == 0)
+  {
+    return MINUTES_PER_DAY * SECONDS_PER_MINUTE;
+  }
+  return retry_time * SECONDS_PER_MINUTE;
+}
+
+u64 NWC24Dl::GetDownloadMargin(u16 entry_index) const
+{
+  return Common::swap16(m_data.entries[entry_index].dl_margin) * SECONDS_PER_MINUTE;
+}
+
+void NWC24Dl::SetNextDownloadTime(u16 record_index, u64 value, std::optional<u8> subtask_id)
+{
+  if (subtask_id)
+  {
+    m_data.entries[record_index].subtask_timestamps[*subtask_id] =
+        Common::swap32(static_cast<u32>(value / SECONDS_PER_MINUTE));
+  }
+
+  m_data.records[record_index].next_dl_timestamp =
+      Common::swap32(static_cast<u32>(value / SECONDS_PER_MINUTE));
+}
+
+u64 NWC24Dl::GetLastSubtaskDownloadTime(u16 entry_index, u8 subtask_id) const
+{
+  return Common::swap32(m_data.entries[entry_index].subtask_timestamps[subtask_id]) *
+             SECONDS_PER_MINUTE +
+         Common::swap32(m_data.entries[entry_index].server_dl_interval) * SECONDS_PER_MINUTE;
+}
+
 u32 NWC24Dl::Magic() const
 {
   return Common::swap32(m_data.header.magic);

Source/Core/Core/IOS/Network/KD/NWC24DL.h

@@ -27,14 +27,24 @@ public:
 
   s32 CheckNwc24DlList() const;
 
-  bool DoesEntryExist(u16 entry_index);
+  bool DoesEntryExist(u16 entry_index) const;
   bool IsEncrypted(u16 entry_index) const;
   bool IsRSASigned(u16 entry_index) const;
+  bool SkipSchedulerDownload(u16 entry_index) const;
+  bool HasSubtask(u16 entry_index) const;
+  bool IsSubtaskDownloadDisabled(u16 entry_index) const;
+  bool IsValidSubtask(u16 entry_index, u8 subtask_id) const;
   std::string GetVFFContentName(u16 entry_index, std::optional<u8> subtask_id) const;
   std::string GetDownloadURL(u16 entry_index, std::optional<u8> subtask_id) const;
   std::string GetVFFPath(u16 entry_index) const;
   WC24PubkMod GetWC24PubkMod(u16 entry_index) const;
 
+  u64 GetNextDownloadTime(u16 record_index) const;
+  u64 GetDownloadMargin(u16 entry_index) const;
+  void SetNextDownloadTime(u16 record_index, u64 value, std::optional<u8> subtask_id);
+  u64 GetRetryTime(u16 entry_index) const;
+  u64 GetLastSubtaskDownloadTime(u16 entry_index, u8 subtask_id) const;
+
   u32 Magic() const;
   void SetMagic(u32 magic);
 
@@ -46,10 +56,12 @@ public:
 private:
   static constexpr u32 DL_LIST_MAGIC = 0x5763446C;  // WcDl
   static constexpr u32 MAX_SUBENTRIES = 32;
+  static constexpr u64 SECONDS_PER_MINUTE = 60;
+  static constexpr u64 MINUTES_PER_DAY = 1440;
 
   enum EntryType : u8
   {
-    UNK = 1,
+    SUBTASK = 1,
     MAIL,
     CHANNEL_CONTENT,
     UNUSED = 0xff
@@ -91,15 +103,14 @@ private:
     u16 padding1;
     u16 remaining_downloads;
     u16 error_count;
-    u16 dl_frequency;
-    u16 dl_frequency_when_err;
+    u16 dl_margin;
+    u16 retry_frequency;
     s32 error_code;
     u8 subtask_id;
     u8 subtask_type;
-    u8 subtask_flags;
-    u8 padding2;
+    u16 subtask_flags;
     u32 subtask_bitmask;
-    s32 unknown2;
+    u32 server_dl_interval;
     u32 dl_timestamp;  // Last DL time
     u32 subtask_timestamps[32];
     char dl_url[236];

Source/Core/Core/IOS/Network/KD/NetKDRequest.cpp

@@ -23,6 +23,7 @@
 #include "Core/CommonTitles.h"
 #include "Core/HW/Memmap.h"
 #include "Core/IOS/FS/FileSystem.h"
+#include "Core/IOS/Network/KD/NetKDTime.h"
 #include "Core/IOS/Network/KD/VFF/VFFUtil.h"
 #include "Core/IOS/Network/Socket.h"
 #include "Core/IOS/Uids.h"
@@ -239,8 +240,20 @@ void NetKDRequestDevice::SchedulerWorker(const SchedulerEvent event)
 {
   if (event == SchedulerEvent::Download)
   {
-    // TODO: Implement downloader part of scheduler
-    return;
+    u16 entry_index{};
+    std::optional<u8> subtask_id{};
+    NWC24::ErrorCode code = DetermineDownloadTask(&entry_index, &subtask_id);
+    if (code != NWC24::WC24_OK)
+    {
+      LogError(ErrorType::KD_Download, code);
+      return;
+    }
+
+    code = KDDownload(entry_index, subtask_id);
+    if (code != NWC24::WC24_OK)
+    {
+      LogError(ErrorType::KD_Download, code);
+    }
   }
   else
   {
@@ -400,9 +413,109 @@ NWC24::ErrorCode NetKDRequestDevice::KDCheckMail(u32* mail_flag, u32* interval)
   return NWC24::WC24_OK;
 }
 
+NWC24::ErrorCode NetKDRequestDevice::DetermineDownloadTask(u16* entry_index,
+                                                           std::optional<u8>* subtask_id) const
+{
+  // As the scheduler does not tell us which entry to download, we must determine that.
+  // A correct entry is one that hasn't been downloaded the longest compared to other entries.
+  // We first need current UTC.
+  const auto time_device =
+      std::static_pointer_cast<NetKDTimeDevice>(GetIOS()->GetDeviceByName("/dev/net/kd/time"));
+  const u64 current_utc = time_device->GetAdjustedUTC();
+  u64 lowest_timestamp = std::numeric_limits<u64>::max();
+
+  for (u16 i = 0; i < static_cast<u16>(NWC24::NWC24Dl::MAX_ENTRIES); i++)
+  {
+    if (!m_dl_list.DoesEntryExist(i))
+      continue;
+
+    if (m_dl_list.SkipSchedulerDownload(i))
+      continue;
+
+    const u64 next_dl_time = m_dl_list.GetNextDownloadTime(i);
+
+    // First determine if UTC is greater than the next download time.
+    if (current_utc < next_dl_time)
+      continue;
+
+    // If this task's next download time is less than the lowest_timestamp, this is the task we
+    // want. However, we must determine if this has a subtask and wants to be downloaded.
+    if (next_dl_time < lowest_timestamp)
+    {
+      if (m_dl_list.HasSubtask(i))
+      {
+        NWC24::ErrorCode code = DetermineSubtask(i, subtask_id);
+        if (code != NWC24::WC24_OK)
+        {
+          // No valid subtask found or downloading is disabled.
+          continue;
+        }
+      }
+
+      lowest_timestamp = next_dl_time;
+      *entry_index = i;
+    }
+  }
+
+  // Determine if we actually found an entry to download.
+  if (lowest_timestamp == std::numeric_limits<u64>::max())
+    return NWC24::WC24_ERR_NOT_FOUND;
+
+  return NWC24::WC24_OK;
+}
+
+NWC24::ErrorCode NetKDRequestDevice::DetermineSubtask(u16 entry_index,
+                                                      std::optional<u8>* subtask_id) const
+{
+  // Before we do anything, determine if this task wants to be downloaded
+  if (m_dl_list.IsSubtaskDownloadDisabled(entry_index))
+    return NWC24::WC24_ERR_DISABLED;
+
+  const auto time_device =
+      std::static_pointer_cast<NetKDTimeDevice>(GetIOS()->GetDeviceByName("/dev/net/kd/time"));
+  const u64 current_utc = time_device->GetAdjustedUTC();
+  for (u8 i = 0; i < 32; i++)
+  {
+    if (!m_dl_list.IsValidSubtask(entry_index, i))
+      continue;
+
+    // Unlike DetermineDownloadTask, DetermineSubtask gets the first download time lower than UTC.
+    const u64 last_download_time = m_dl_list.GetLastSubtaskDownloadTime(entry_index, i);
+    if (last_download_time < current_utc)
+    {
+      *subtask_id = i;
+      return NWC24::WC24_OK;
+    }
+  }
+
+  return NWC24::WC24_ERR_INVALID_VALUE;
+}
+
 NWC24::ErrorCode NetKDRequestDevice::KDDownload(const u16 entry_index,
                                                 const std::optional<u8> subtask_id)
 {
+  bool success = false;
+  Common::ScopeGuard state_guard([&] {
+    const auto time_device =
+        std::static_pointer_cast<NetKDTimeDevice>(GetIOS()->GetDeviceByName("/dev/net/kd/time"));
+    const u64 current_utc = time_device->GetAdjustedUTC();
+    if (success)
+    {
+      // Set the next download time to the dl_margin
+      m_dl_list.SetNextDownloadTime(
+          entry_index, current_utc + m_dl_list.GetDownloadMargin(entry_index), subtask_id);
+    }
+    else
+    {
+      // Else set it to the retry margin
+      m_dl_list.SetNextDownloadTime(entry_index, current_utc + m_dl_list.GetRetryTime(entry_index),
+                                    subtask_id);
+    }
+
+    // Finally flush
+    m_dl_list.WriteDlList();
+  });
+
   std::vector<u8> file_data;
 
   // Content metadata
@@ -492,8 +605,12 @@ NWC24::ErrorCode NetKDRequestDevice::KDDownload(const u16 entry_index,
                                                     m_ios.GetFS(), file_data);
 
   if (reply != NWC24::WC24_OK)
+  {
     LogError(ErrorType::KD_Download, reply);
+    return reply;
+  }
 
+  success = true;
   return reply;
 }
 

Source/Core/Core/IOS/Network/KD/NetKDRequest.h

@@ -85,6 +85,8 @@ private:
   void LogError(ErrorType error_type, s32 error_code);
   void SchedulerTimer();
   void SchedulerWorker(SchedulerEvent event);
+  NWC24::ErrorCode DetermineDownloadTask(u16* entry_index, std::optional<u8>* subtask_id) const;
+  NWC24::ErrorCode DetermineSubtask(u16 entry_index, std::optional<u8>* subtask_id) const;
 
   static std::string GetValueFromCGIResponse(const std::string& response, const std::string& key);
   static constexpr std::array<u8, 20> MAIL_CHECK_KEY = {0xce, 0x4c, 0xf2, 0x9a, 0x3d, 0x6b, 0xe1,

Source/Core/Core/IOS/Network/KD/NetKDTime.h

@@ -18,15 +18,15 @@ public:
 
   std::optional<IPCReply> IOCtl(const IOCtlRequest& request) override;
 
+  // Returns seconds since Wii epoch
+  // +/- any bias set from IOCTL_NW24_SET_UNIVERSAL_TIME
+  u64 GetAdjustedUTC() const;
+
 private:
   // TODO: depending on CEXIIPL is a hack which I don't feel like
   // removing because the function itself is pretty hackish;
   // wait until I re-port my netplay rewrite
 
-  // Returns seconds since Wii epoch
-  // +/- any bias set from IOCTL_NW24_SET_UNIVERSAL_TIME
-  u64 GetAdjustedUTC() const;
-
   // Store the difference between what the Wii thinks is UTC and
   // what the host OS thinks
   void SetAdjustedUTC(u64 wii_utc);