CImage.cpp 109 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513151415151516151715181519152015211522152315241525152615271528152915301531153215331534153515361537153815391540154115421543154415451546154715481549155015511552155315541555155615571558155915601561156215631564156515661567156815691570157115721573157415751576157715781579158015811582158315841585158615871588158915901591159215931594159515961597159815991600160116021603160416051606160716081609161016111612161316141615161616171618161916201621162216231624162516261627162816291630163116321633163416351636163716381639164016411642164316441645164616471648164916501651165216531654165516561657165816591660166116621663166416651666166716681669167016711672167316741675167616771678167916801681168216831684168516861687168816891690169116921693169416951696169716981699170017011702170317041705170617071708170917101711171217131714171517161717171817191720172117221723172417251726172717281729173017311732173317341735173617371738173917401741174217431744174517461747174817491750175117521753175417551756175717581759176017611762176317641765176617671768176917701771177217731774177517761777177817791780178117821783178417851786178717881789179017911792179317941795179617971798179918001801180218031804180518061807180818091810181118121813181418151816181718181819182018211822182318241825182618271828182918301831183218331834183518361837183818391840184118421843184418451846184718481849185018511852185318541855185618571858185918601861186218631864186518661867186818691870187118721873187418751876187718781879188018811882188318841885188618871888188918901891189218931894189518961897189818991900190119021903190419051906190719081909191019111912191319141915191619171918191919201921192219231924192519261927192819291930193119321933193419351936193719381939194019411942194319441945194619471948194919501951195219531954195519561957195819591960196119621963196419651966196719681969197019711972197319741975197619771978197919801981198219831984198519861987198819891990199119921993199419951996199719981999200020012002200320042005200620072008200920102011201220132014201520162017201820192020202120222023202420252026202720282029203020312032203320342035203620372038203920402041204220432044204520462047204820492050205120522053205420552056205720582059206020612062206320642065206620672068206920702071207220732074207520762077207820792080208120822083208420852086208720882089209020912092209320942095209620972098209921002101210221032104210521062107210821092110211121122113211421152116211721182119212021212122212321242125212621272128212921302131213221332134213521362137213821392140214121422143214421452146214721482149215021512152215321542155215621572158215921602161216221632164216521662167216821692170217121722173217421752176217721782179218021812182218321842185218621872188218921902191219221932194219521962197219821992200220122022203220422052206220722082209221022112212221322142215221622172218221922202221222222232224222522262227222822292230223122322233223422352236223722382239224022412242224322442245224622472248224922502251225222532254225522562257225822592260226122622263226422652266226722682269227022712272227322742275227622772278227922802281228222832284228522862287228822892290229122922293229422952296229722982299230023012302230323042305230623072308230923102311231223132314231523162317231823192320232123222323232423252326232723282329233023312332233323342335233623372338233923402341234223432344234523462347234823492350235123522353235423552356235723582359236023612362236323642365236623672368236923702371237223732374237523762377237823792380238123822383238423852386238723882389239023912392239323942395239623972398239924002401240224032404240524062407240824092410241124122413241424152416241724182419242024212422242324242425242624272428242924302431243224332434243524362437243824392440244124422443244424452446244724482449245024512452245324542455245624572458245924602461246224632464246524662467246824692470247124722473247424752476247724782479248024812482248324842485248624872488248924902491249224932494249524962497249824992500250125022503250425052506250725082509251025112512251325142515251625172518251925202521252225232524252525262527252825292530253125322533253425352536253725382539254025412542254325442545254625472548254925502551255225532554255525562557255825592560256125622563256425652566256725682569257025712572257325742575257625772578257925802581258225832584258525862587258825892590259125922593259425952596259725982599260026012602260326042605260626072608260926102611261226132614261526162617
  1. #include "CImage.h"
  2. #include "CRenderable.h"
  3. #include <algorithm>
  4. #include <array>
  5. #include <cstdio>
  6. #include <cstring>
  7. #include <iterator>
  8. #include <limits>
  9. #include <optional>
  10. #include <sstream>
  11. #include <vector>
  12. #include <glm/glm.hpp>
  13. #include <glm/gtc/matrix_transform.hpp>
  14. #define GLM_ENABLE_EXPERIMENTAL
  15. #include <glm/gtx/rotate_vector.hpp>
  16. #undef GLM_ENABLE_EXPERIMENTAL
  17. #include "WallpaperEngine/Data/Model/DynamicValue.h"
  18. #include "WallpaperEngine/Data/Model/Material.h"
  19. #include "WallpaperEngine/Data/Model/Object.h"
  20. #include "WallpaperEngine/Data/Model/UserSetting.h"
  21. #include "WallpaperEngine/Data/Parsers/MaterialParser.h"
  22. #include "WallpaperEngine/Data/Utils/BinaryReader.h"
  23. #include "WallpaperEngine/Data/Utils/MemoryStream.h"
  24. #include "WallpaperEngine/Logging/Log.h"
  25. using namespace WallpaperEngine;
  26. using namespace WallpaperEngine::Render::Objects;
  27. using namespace WallpaperEngine::Render::Objects::Effects;
  28. using namespace WallpaperEngine::Data::Parsers;
  29. using namespace WallpaperEngine::Data::Builders;
  30. using namespace WallpaperEngine::Data::Utils;
  31. extern float g_Time;
  32. namespace {
  33. glm::vec2 rotateVec2 (const glm::vec2& value, float angle) {
  34. const float cosAngle = std::cos (angle);
  35. const float sinAngle = std::sin (angle);
  36. return { value.x * cosAngle - value.y * sinAngle, value.x * sinAngle + value.y * cosAngle };
  37. }
  38. bool isMagentaNeonTint (const glm::vec3& color) { return color.r > 0.55f && color.g < 0.25f && color.b > 0.45f; }
  39. std::optional<glm::vec3> findMagentaCompositeTint (const Image& image, const std::vector<int>& skippedEffectIds) {
  40. for (const auto& effect : image.effects) {
  41. if (std::find (skippedEffectIds.begin (), skippedEffectIds.end (), static_cast<int> (effect->id))
  42. != skippedEffectIds.end ()) {
  43. continue;
  44. }
  45. if (!effect->visible->value->getBool ()) {
  46. continue;
  47. }
  48. for (const auto& passOverride : effect->passOverrides) {
  49. const auto compositeCombo = passOverride->combos.find ("COMPOSITE");
  50. if (compositeCombo == passOverride->combos.end () || compositeCombo->second != 2) {
  51. continue;
  52. }
  53. const auto compositeColor = passOverride->constants.find ("compositecolor");
  54. if (compositeColor == passOverride->constants.end () || compositeColor->second == nullptr
  55. || compositeColor->second->value == nullptr) {
  56. continue;
  57. }
  58. const auto tint = compositeColor->second->value->getVec3 ();
  59. if (isMagentaNeonTint (tint)) {
  60. return tint;
  61. }
  62. }
  63. }
  64. return std::nullopt;
  65. }
  66. struct PuppetMeshBlock {
  67. size_t headerOffset = 0;
  68. uint32_t vertexBytes = 0;
  69. uint32_t indexBytes = 0;
  70. };
  71. // Finds every byte offset that could plausibly be a MDLV mesh header (DWORD vertexByteLength
  72. // immediately followed by that many bytes of vertex data, then a DWORD indexByteLength followed by
  73. // that many bytes of indices, all landing before the MDLS block). This intentionally doesn't know or
  74. // care about the per-vertex stride - that's resolved afterwards against whatever candidates come back,
  75. // since the stride isn't reliably predictable from the MDLV header version alone (see
  76. // resolvePuppetVertexLayout).
  77. std::vector<PuppetMeshBlock>
  78. findPuppetMeshBlockCandidates (const BinaryReader& reader, size_t markerSize, size_t mdlsOffset, size_t meshHeaderSize) {
  79. std::vector<PuppetMeshBlock> candidates;
  80. for (size_t offset = markerSize; offset + meshHeaderSize + sizeof (uint32_t) < mdlsOffset; offset++) {
  81. reader.base ().seekg (static_cast<std::streamoff> (offset + sizeof (uint32_t)), std::ios::beg);
  82. const uint32_t candidateVertexBytes = reader.nextUInt32 ();
  83. const size_t verticesOffset = offset + meshHeaderSize;
  84. const size_t indexLengthOffset = verticesOffset + candidateVertexBytes;
  85. if (candidateVertexBytes == 0 || indexLengthOffset + sizeof (uint32_t) > mdlsOffset) {
  86. continue;
  87. }
  88. reader.base ().seekg (static_cast<std::streamoff> (indexLengthOffset), std::ios::beg);
  89. const uint32_t candidateIndexBytes = reader.nextUInt32 ();
  90. const size_t indicesOffset = indexLengthOffset + sizeof (uint32_t);
  91. if (candidateIndexBytes == 0 || candidateIndexBytes % (sizeof (uint16_t) * 3) != 0
  92. || indicesOffset + candidateIndexBytes > mdlsOffset) {
  93. continue;
  94. }
  95. candidates.push_back (
  96. PuppetMeshBlock { .headerOffset = offset, .vertexBytes = candidateVertexBytes, .indexBytes = candidateIndexBytes }
  97. );
  98. }
  99. return candidates;
  100. }
  101. struct PuppetVertexLayout {
  102. PuppetMeshBlock block;
  103. size_t vertexStride = 0;
  104. size_t uvOffset = 0;
  105. };
  106. // Reads raw positions/UVs/indices for a candidate (block, stride) pair. The UV pair has only ever
  107. // been observed as the trailing 8 bytes of the vertex record, whatever bone/weight data precedes it
  108. // (position(12) + ... + uv(8)), so uvOffset = stride - 8 throughout.
  109. struct PuppetMeshData {
  110. std::vector<GLfloat> positions;
  111. std::vector<GLfloat> texcoords;
  112. std::vector<GLushort> indices;
  113. };
  114. std::optional<PuppetMeshData>
  115. readPuppetMeshData (const BinaryReader& reader, const PuppetMeshBlock& block, size_t meshHeaderSize, size_t vertexStride) {
  116. if (block.vertexBytes % vertexStride != 0) {
  117. return std::nullopt;
  118. }
  119. const size_t uvOffset = vertexStride - sizeof (GLfloat) * 2;
  120. const size_t vertexCount = block.vertexBytes / vertexStride;
  121. const size_t verticesOffset = block.headerOffset + meshHeaderSize;
  122. const size_t indicesOffset = verticesOffset + block.vertexBytes + sizeof (uint32_t);
  123. const size_t indexCount = block.indexBytes / sizeof (uint16_t);
  124. PuppetMeshData data;
  125. data.positions.reserve (vertexCount * 3);
  126. data.texcoords.reserve (vertexCount * 2);
  127. data.indices.reserve (indexCount);
  128. for (size_t index = 0; index < vertexCount; index++) {
  129. const size_t vertexOffset = verticesOffset + index * vertexStride;
  130. reader.base ().seekg (static_cast<std::streamoff> (vertexOffset), std::ios::beg);
  131. const float x = reader.nextFloat ();
  132. const float y = reader.nextFloat ();
  133. const float z = reader.nextFloat ();
  134. reader.base ().seekg (static_cast<std::streamoff> (vertexOffset + uvOffset), std::ios::beg);
  135. const float u = reader.nextFloat ();
  136. const float v = reader.nextFloat ();
  137. data.positions.push_back (x);
  138. data.positions.push_back (y);
  139. data.positions.push_back (z);
  140. data.texcoords.push_back (u);
  141. data.texcoords.push_back (v);
  142. }
  143. reader.base ().seekg (static_cast<std::streamoff> (indicesOffset), std::ios::beg);
  144. for (size_t index = 0; index < indexCount; index++) {
  145. uint16_t value = 0;
  146. reader.next (reinterpret_cast<char*> (&value), sizeof (value));
  147. if (value >= vertexCount) {
  148. return std::nullopt;
  149. }
  150. data.indices.push_back (value);
  151. }
  152. return data;
  153. }
  154. // Blend indices/weights are always the 32 bytes immediately before the UV pair, regardless of stride
  155. // (see docs/rendering/MDL_FILES.md) - position(12) + [normal(12) + tangent4(16), wide format only] +
  156. // blendindices(16) + blendweight(16) + uv(8).
  157. struct PuppetBlendData {
  158. std::vector<glm::uvec4> indices;
  159. std::vector<glm::vec4> weights;
  160. };
  161. std::optional<PuppetBlendData> readPuppetBlendData (
  162. const BinaryReader& reader, const PuppetMeshBlock& block, size_t meshHeaderSize, size_t vertexStride
  163. ) {
  164. if (vertexStride < 40 || block.vertexBytes % vertexStride != 0) {
  165. return std::nullopt;
  166. }
  167. const size_t blendIndicesOffset = vertexStride - 40;
  168. const size_t blendWeightsOffset = vertexStride - 24;
  169. const size_t vertexCount = block.vertexBytes / vertexStride;
  170. const size_t verticesOffset = block.headerOffset + meshHeaderSize;
  171. PuppetBlendData data;
  172. data.indices.reserve (vertexCount);
  173. data.weights.reserve (vertexCount);
  174. for (size_t index = 0; index < vertexCount; index++) {
  175. const size_t vertexOffset = verticesOffset + index * vertexStride;
  176. reader.base ().seekg (static_cast<std::streamoff> (vertexOffset + blendIndicesOffset), std::ios::beg);
  177. glm::uvec4 boneIndices;
  178. boneIndices.x = reader.nextUInt32 ();
  179. boneIndices.y = reader.nextUInt32 ();
  180. boneIndices.z = reader.nextUInt32 ();
  181. boneIndices.w = reader.nextUInt32 ();
  182. reader.base ().seekg (static_cast<std::streamoff> (vertexOffset + blendWeightsOffset), std::ios::beg);
  183. glm::vec4 boneWeights;
  184. boneWeights.x = reader.nextFloat ();
  185. boneWeights.y = reader.nextFloat ();
  186. boneWeights.z = reader.nextFloat ();
  187. boneWeights.w = reader.nextFloat ();
  188. data.indices.push_back (boneIndices);
  189. data.weights.push_back (boneWeights);
  190. }
  191. return data;
  192. }
  193. // Scores how plausible a candidate vertex layout is: real puppet meshes are triangulated warp grids,
  194. // so triangles formed by adjacent indices should be small relative to the mesh's overall size. A wrong
  195. // stride reinterprets bone/weight bytes as positions, which decorrelates neighbouring vertices and
  196. // produces comparatively huge, inconsistent triangles. Lower is better; nullopt if unscorable (e.g. a
  197. // degenerate single-point mesh).
  198. std::optional<double> scorePuppetMeshCoherence (const PuppetMeshData& data) {
  199. if (data.indices.size () < 3) {
  200. return std::nullopt;
  201. }
  202. glm::vec3 min (std::numeric_limits<float>::max ());
  203. glm::vec3 max (std::numeric_limits<float>::lowest ());
  204. const size_t vertexCount = data.positions.size () / 3;
  205. for (size_t i = 0; i < vertexCount; i++) {
  206. const glm::vec3 p (data.positions[i * 3], data.positions[i * 3 + 1], data.positions[i * 3 + 2]);
  207. min = glm::min (min, p);
  208. max = glm::max (max, p);
  209. }
  210. const double diagonal = glm::length (max - min);
  211. if (diagonal <= 0.0) {
  212. return std::nullopt;
  213. }
  214. double totalEdgeLength = 0.0;
  215. size_t edgeCount = 0;
  216. for (size_t triangle = 0; triangle + 2 < data.indices.size (); triangle += 3) {
  217. const auto vertexPosition = [&data] (size_t index) {
  218. return glm::vec3 (data.positions[index * 3], data.positions[index * 3 + 1], data.positions[index * 3 + 2]);
  219. };
  220. const glm::vec3 a = vertexPosition (data.indices[triangle]);
  221. const glm::vec3 b = vertexPosition (data.indices[triangle + 1]);
  222. const glm::vec3 c = vertexPosition (data.indices[triangle + 2]);
  223. totalEdgeLength += glm::length (a - b) + glm::length (b - c) + glm::length (c - a);
  224. edgeCount += 3;
  225. }
  226. if (edgeCount == 0) {
  227. return std::nullopt;
  228. }
  229. return (totalEdgeLength / static_cast<double> (edgeCount)) / diagonal;
  230. }
  231. // The MDLV vertex layout isn't reliably predictable from the header version number alone - the same
  232. // version (e.g. MDLV0023) has been observed with different per-vertex strides depending on how many
  233. // bone influences a given puppet part carries. So instead of a fixed version->stride table, every
  234. // plausible stride is tried against every candidate mesh header found in the file, and whichever
  235. // combination produces the most coherent triangulated mesh wins.
  236. std::optional<PuppetVertexLayout> resolvePuppetVertexLayout (
  237. const BinaryReader& reader, size_t markerSize, size_t mdlsOffset, size_t meshHeaderSize
  238. ) {
  239. constexpr size_t minVertexStride = 20; // position (12 bytes) + uv (8 bytes), no bone data at all
  240. constexpr size_t maxVertexStride = 256; // generous upper bound, comfortably covers multi-bone rigs
  241. constexpr size_t strideStep = 4; // every field observed so far is a 4-byte float/uint
  242. const auto candidates = findPuppetMeshBlockCandidates (reader, markerSize, mdlsOffset, meshHeaderSize);
  243. std::optional<PuppetVertexLayout> best;
  244. double bestScore = std::numeric_limits<double>::max ();
  245. for (const auto& block : candidates) {
  246. for (size_t stride = minVertexStride; stride <= maxVertexStride; stride += strideStep) {
  247. const auto data = readPuppetMeshData (reader, block, meshHeaderSize, stride);
  248. if (!data.has_value ()) {
  249. continue;
  250. }
  251. const auto score = scorePuppetMeshCoherence (*data);
  252. if (!score.has_value ()) {
  253. continue;
  254. }
  255. if (*score >= bestScore) {
  256. continue;
  257. }
  258. bestScore = *score;
  259. best = PuppetVertexLayout { .block = block, .vertexStride = stride, .uvOffset = stride - sizeof (GLfloat) * 2 };
  260. }
  261. }
  262. return best;
  263. }
  264. struct PuppetBoneSet {
  265. std::vector<PuppetBone> bones;
  266. // Points at whatever section comes right after MDLS's second bone array: MDLA directly for
  267. // puppets with no attachment points, or MDAT (attachment points) otherwise - the caller has to
  268. // check which one it actually is.
  269. size_t nextSectionOffset = 0;
  270. };
  271. // Parses the MDLS section's first bone array (local bind-pose transforms + parent hierarchy). The
  272. // second bone array isn't decoded: its per-bone "name" slot turns out to hold physics/jiggle constraint
  273. // parameters (angle limits, stiffness, a target position) rather than anything about mesh skinning, and
  274. // inverse-bind matrices can be derived from the first array alone by walking the parent chain anyway.
  275. PuppetBoneSet parsePuppetBones (const BinaryReader& reader, size_t mdlsOffset) {
  276. reader.base ().seekg (static_cast<std::streamoff> (mdlsOffset), std::ios::beg);
  277. char header[9];
  278. reader.next (header, sizeof (header));
  279. const uint32_t nextSectionOffset = reader.nextUInt32 ();
  280. const uint32_t boneCount = reader.nextUInt32 ();
  281. // A bone count this large can only be a garbage read (wrong mdlsOffset or an unrecognized MDLS
  282. // layout), not a real rig. Same reasoning as the clip/point-count guards below.
  283. constexpr uint32_t maxPlausibleBoneCount = 512;
  284. if (boneCount > maxPlausibleBoneCount) {
  285. sLog.error ("Puppet bone count (", boneCount, ") looks implausible, skipping puppet mesh skinning");
  286. return {};
  287. }
  288. PuppetBoneSet result;
  289. result.nextSectionOffset = nextSectionOffset;
  290. result.bones.reserve (boneCount);
  291. for (uint32_t i = 0; i < boneCount; i++) {
  292. // records start with a null-terminated name, empty for most rigs
  293. (void) reader.nextNullTerminatedString ();
  294. (void) reader.nextUInt32 (); // type, unused
  295. const int parent = reader.nextInt ();
  296. const uint32_t matrixBytes = reader.nextUInt32 ();
  297. glm::mat4 bindLocal (1.0f);
  298. if (matrixBytes == sizeof (float) * 16) {
  299. float m[16];
  300. for (float& value : m) {
  301. value = reader.nextFloat ();
  302. }
  303. // the file stores a row-vector-convention, row-major matrix; feeding the 16 values straight
  304. // into glm's column-major constructor produces exactly its transpose, which is the
  305. // column-vector matrix glm needs to compute M * v
  306. bindLocal = glm::mat4 (
  307. m[0], m[1], m[2], m[3], m[4], m[5], m[6], m[7], m[8], m[9], m[10], m[11], m[12], m[13], m[14], m[15]
  308. );
  309. } else {
  310. // an implausible byte count here means this bone record wasn't decoded correctly; bail out
  311. // rather than seeking by an untrusted amount and reading whatever garbage follows as bones
  312. constexpr uint32_t maxPlausibleMatrixBytes = 4096;
  313. if (matrixBytes > maxPlausibleMatrixBytes) {
  314. sLog.error (
  315. "Puppet bone ", i, " has an implausible matrix byte count (", matrixBytes,
  316. "), stopping here (", result.bones.size (), " bone(s) kept)"
  317. );
  318. break;
  319. }
  320. reader.base ().seekg (static_cast<std::streamoff> (matrixBytes), std::ios::cur);
  321. }
  322. // trailing per-bone string, jiggle/physics JSON for some rigs
  323. (void) reader.nextNullTerminatedString ();
  324. result.bones.push_back (PuppetBone { .parent = parent, .bindLocal = bindLocal });
  325. }
  326. return result;
  327. }
  328. // Resolves each bone's world transform from its parent-relative local transform, by walking up
  329. // the parent chain rather than assuming the array is stored parent-before-child. Nothing in the MDL
  330. // format guarantees that ordering, and it does not hold for every rig seen in practice (small
  331. // sub-meshes like a puppet's eyes/eyebrows in particular) - treating an out-of-order parent as "not
  332. // yet resolved" instead of silently falling back to "no parent" is what makes a bone whose parent
  333. // happens to sit later in the array compose correctly instead of coming out at raw bone-local
  334. // coordinates, detached from the rest of the rig it's supposed to be attached to.
  335. void resolveBoneWorldTransform (
  336. size_t index, const std::vector<int>& parents, const std::vector<glm::mat4>& locals, std::vector<glm::mat4>& world,
  337. std::vector<bool>& resolved, std::vector<bool>& visiting
  338. ) {
  339. if (resolved[index]) {
  340. return;
  341. }
  342. const int parent = parents[index];
  343. // a missing parent, an out-of-range index, or a cycle back onto a bone still being resolved are
  344. // all treated the same way a genuine root bone would be: no parent transform to fold in
  345. if (parent < 0 || static_cast<size_t> (parent) >= parents.size () || visiting[index]) {
  346. world[index] = locals[index];
  347. } else {
  348. visiting[index] = true;
  349. resolveBoneWorldTransform (static_cast<size_t> (parent), parents, locals, world, resolved, visiting);
  350. visiting[index] = false;
  351. world[index] = world[static_cast<size_t> (parent)] * locals[index];
  352. }
  353. resolved[index] = true;
  354. }
  355. std::vector<glm::mat4> composeBoneWorldTransforms (const std::vector<int>& parents, const std::vector<glm::mat4>& locals) {
  356. std::vector<glm::mat4> world (locals.size ());
  357. std::vector<bool> resolved (locals.size (), false);
  358. std::vector<bool> visiting (locals.size (), false);
  359. for (size_t i = 0; i < locals.size (); i++) {
  360. resolveBoneWorldTransform (i, parents, locals, world, resolved, visiting);
  361. }
  362. return world;
  363. }
  364. struct PuppetAttachmentPointSet {
  365. std::vector<PuppetAttachmentPoint> points;
  366. size_t mdlaOffset = 0;
  367. };
  368. // Parses the optional MDAT section (named attachment points other objects can follow, e.g.
  369. // scene.json's "attachment": "orb" - see docs/rendering/MDL_FILES.md). Stops - keeping whatever
  370. // points parsed cleanly so far - the moment an entry looks implausible, since only two real point
  371. // names have been confirmed against real data and the tail of this section isn't fully understood.
  372. PuppetAttachmentPointSet parsePuppetAttachmentPoints (const BinaryReader& reader, size_t mdatOffset, uint32_t boneCount) {
  373. reader.base ().seekg (static_cast<std::streamoff> (mdatOffset), std::ios::beg);
  374. char header[9];
  375. reader.next (header, sizeof (header));
  376. PuppetAttachmentPointSet result;
  377. result.mdlaOffset = reader.nextUInt32 ();
  378. uint16_t pointCount = 0;
  379. reader.next (reinterpret_cast<char*> (&pointCount), sizeof (pointCount));
  380. // What looks like a fixed WORD trailing every point's matrix is actually the NEXT point's bone
  381. // index, one slot early: point 0's bone index lives right here, straight after pointCount (this
  382. // field was previously assumed to be padding/unused), and each point's own trailing WORD belongs
  383. // to the point after it - which is why the last point has no trailing WORD at all. Confirmed on
  384. // real puppet data: reading a trailing WORD for every point (including the last) overran two bytes
  385. // past the MDAT section's own declared length, landing exactly on the next section's magic bytes;
  386. // this shifted reading consumes the section's declared byte length exactly, with nothing left over.
  387. uint16_t nextBoneIndex = 0;
  388. reader.next (reinterpret_cast<char*> (&nextBoneIndex), sizeof (nextBoneIndex));
  389. constexpr uint16_t maxPlausiblePointCount = 256;
  390. if (pointCount > maxPlausiblePointCount) {
  391. sLog.error ("Puppet attachment point count (", pointCount, ") looks implausible, ignoring attachment points");
  392. return result;
  393. }
  394. for (uint16_t i = 0; i < pointCount; i++) {
  395. const std::string name = reader.nextNullTerminatedString ();
  396. float m[16];
  397. for (float& value : m) {
  398. value = reader.nextFloat ();
  399. }
  400. const uint16_t boneIndex = nextBoneIndex;
  401. if (i + 1 < pointCount) {
  402. reader.next (reinterpret_cast<char*> (&nextBoneIndex), sizeof (nextBoneIndex));
  403. }
  404. if (name.empty () || boneIndex >= boneCount) {
  405. sLog.error (
  406. "Puppet attachment point ", i, " (name=", name, ", bone=", boneIndex,
  407. ") looks implausible, stopping here (", result.points.size (), " point(s) kept)"
  408. );
  409. break;
  410. }
  411. // same row-major-to-column-major transpose trick used for PuppetBone::bindLocal
  412. const glm::mat4 localTransform (
  413. m[0], m[1], m[2], m[3], m[4], m[5], m[6], m[7], m[8], m[9], m[10], m[11], m[12], m[13], m[14], m[15]
  414. );
  415. result.points.push_back (
  416. PuppetAttachmentPoint { .name = name, .boneIndex = boneIndex, .localTransform = localTransform }
  417. );
  418. }
  419. return result;
  420. }
  421. // Looks ahead from searchStart for the next byte offset that looks like a valid clip header, to
  422. // resynchronize past the still-undecoded per-clip trailer when a MDLA section holds more than one clip.
  423. std::optional<size_t> findNextPuppetClipHeader (
  424. const std::vector<char>& data, size_t searchStart, size_t searchLimit, uint32_t expectedBoneCount
  425. ) {
  426. const auto readCString = [&data] (size_t& cursor) -> std::optional<std::string> {
  427. const size_t start = cursor;
  428. while (cursor < data.size () && data[cursor] != 0) {
  429. const auto byte = static_cast<unsigned char> (data[cursor]);
  430. if (byte < 0x20 || byte > 0x7e || cursor - start > 64) {
  431. return std::nullopt;
  432. }
  433. cursor++;
  434. }
  435. if (cursor >= data.size () || cursor == start) {
  436. return std::nullopt;
  437. }
  438. std::string value (data.data () + start, cursor - start);
  439. cursor++;
  440. return value;
  441. };
  442. for (size_t offset = searchStart; offset < searchLimit; offset++) {
  443. size_t cursor = offset;
  444. if (!readCString (cursor).has_value () || !readCString (cursor).has_value ()) {
  445. continue;
  446. }
  447. if (cursor + 16 > data.size ()) {
  448. continue;
  449. }
  450. float fps;
  451. uint32_t frameCount;
  452. uint32_t flag;
  453. uint32_t boneCount;
  454. std::memcpy (&fps, data.data () + cursor, sizeof (fps));
  455. std::memcpy (&frameCount, data.data () + cursor + 4, sizeof (frameCount));
  456. std::memcpy (&flag, data.data () + cursor + 8, sizeof (flag));
  457. std::memcpy (&boneCount, data.data () + cursor + 12, sizeof (boneCount));
  458. if (fps >= 1.0f && fps <= 240.0f && frameCount >= 1 && frameCount <= 100000 && flag == 0
  459. && boneCount == expectedBoneCount) {
  460. return offset;
  461. }
  462. }
  463. return std::nullopt;
  464. }
  465. // Parses every baked animation clip out of the MDLA section (see docs/rendering/MDL_FILES.md).
  466. std::vector<PuppetAnimationClip> parsePuppetAnimationClips (
  467. const std::vector<char>& data, const BinaryReader& reader, size_t mdlaOffset, uint32_t expectedBoneCount,
  468. bool dumpBone29
  469. ) {
  470. reader.base ().seekg (static_cast<std::streamoff> (mdlaOffset), std::ios::beg);
  471. char header[9];
  472. reader.next (header, sizeof (header));
  473. (void) reader.nextUInt32 (); // total content size, unused
  474. const uint32_t clipCount = reader.nextUInt32 ();
  475. (void) reader.nextUInt32 (); // ambiguous animation id when there's more than one clip; matched by name instead
  476. (void) reader.nextUInt32 (); // unused, always 0 in every sample seen
  477. // this whole section is only trustworthy insofar as the MDLS "mdlaOffset" field that got us here
  478. // actually landed on a real MDLA layout for this MDLV sub-format - it's only been confirmed against
  479. // MDLV0023 samples so far. A clip count this large can only be a garbage read, not a real file.
  480. constexpr uint32_t maxPlausibleClipCount = 64;
  481. if (clipCount > maxPlausibleClipCount) {
  482. sLog.error (
  483. "Puppet animation clip count (", clipCount, ") looks implausible, assuming this puppet's MDLA layout wasn't "
  484. "recognized and skipping animation entirely"
  485. );
  486. return {};
  487. }
  488. std::vector<PuppetAnimationClip> clips;
  489. clips.reserve (clipCount);
  490. for (uint32_t clipIndex = 0; clipIndex < clipCount; clipIndex++) {
  491. PuppetAnimationClip clip;
  492. clip.name = reader.nextNullTerminatedString ();
  493. clip.mode = reader.nextNullTerminatedString ();
  494. clip.fps = reader.nextFloat ();
  495. clip.frameCount = reader.nextUInt32 ();
  496. (void) reader.nextUInt32 (); // unused, always 0 in every sample seen
  497. const uint32_t boneCount = reader.nextUInt32 ();
  498. constexpr uint32_t maxPlausibleFrameCount = 100000;
  499. if (clip.frameCount > maxPlausibleFrameCount || boneCount != expectedBoneCount) {
  500. sLog.error (
  501. "Puppet animation clip ", clipIndex, " has an implausible frame/bone count (frames=", clip.frameCount,
  502. ", bones=", boneCount, ", expected ", expectedBoneCount,
  503. "), assuming this puppet's MDLA layout wasn't recognized and stopping here"
  504. );
  505. break;
  506. }
  507. clip.boneTracks.resize (boneCount);
  508. for (uint32_t boneIndex = 0; boneIndex < boneCount; boneIndex++) {
  509. (void) reader.nextUInt32 (); // separator, always 0 in every sample seen
  510. const uint32_t trackBytes = reader.nextUInt32 ();
  511. const uint32_t sampleCount = clip.frameCount + 1;
  512. const uint32_t expectedBytes = sampleCount * 9 * sizeof (float);
  513. if (trackBytes != expectedBytes) {
  514. sLog.error (
  515. "Puppet animation track length mismatch in clip ", clip.name, " (expected ", expectedBytes,
  516. ", got ", trackBytes, "), skipping"
  517. );
  518. reader.base ().seekg (static_cast<std::streamoff> (trackBytes), std::ios::cur);
  519. continue;
  520. }
  521. auto& track = clip.boneTracks[boneIndex];
  522. track.reserve (sampleCount);
  523. for (uint32_t sample = 0; sample < sampleCount; sample++) {
  524. PuppetKeyframe keyframe;
  525. keyframe.position = { reader.nextFloat (), reader.nextFloat (), reader.nextFloat () };
  526. keyframe.rotation = { reader.nextFloat (), reader.nextFloat (), reader.nextFloat () };
  527. keyframe.scale = { reader.nextFloat (), reader.nextFloat (), reader.nextFloat () };
  528. track.push_back (keyframe);
  529. }
  530. // TEMP-DIAG: raw keyframe dump for bone 29
  531. if (dumpBone29 && boneIndex == 29 && expectedBoneCount > 29) {
  532. float minRotZ = std::numeric_limits<float>::max (), maxRotZ = std::numeric_limits<float>::lowest ();
  533. float minPosX = std::numeric_limits<float>::max (), maxPosX = std::numeric_limits<float>::lowest ();
  534. for (const auto& kf : track) {
  535. minRotZ = std::min (minRotZ, kf.rotation.z);
  536. maxRotZ = std::max (maxRotZ, kf.rotation.z);
  537. minPosX = std::min (minPosX, kf.position.x);
  538. maxPosX = std::max (maxPosX, kf.position.x);
  539. }
  540. sLog.out (
  541. "TEMP-DIAG bone29 raw track for clip '", clip.name, "': samples=", track.size (), " rotZ=[",
  542. minRotZ, ",", maxRotZ, "] posX=[", minPosX, ",", maxPosX, "] first3=(",
  543. track.size () > 0 ? track[0].rotation.z : 0.0f, ",", track.size () > 1 ? track[1].rotation.z : 0.0f,
  544. ",", track.size () > 2 ? track[2].rotation.z : 0.0f, ") mid=(",
  545. track.size () > 90 ? track[90].rotation.z : 0.0f, ")"
  546. );
  547. }
  548. }
  549. clips.push_back (std::move (clip));
  550. if (clipIndex + 1 < clipCount) {
  551. const auto pos = static_cast<size_t> (reader.base ().tellg ());
  552. const auto next = findNextPuppetClipHeader (data, pos, std::min (pos + 16384, data.size ()), expectedBoneCount);
  553. if (!next.has_value ()) {
  554. sLog.error ("Could not resynchronize puppet animation data after clip ", clips.back ().name);
  555. break;
  556. }
  557. reader.base ().seekg (static_cast<std::streamoff> (*next), std::ios::beg);
  558. }
  559. }
  560. return clips;
  561. }
  562. }
  563. CImage::ResolvedTransform CImage::localTransform (const Object& object) {
  564. glm::vec3 origin = object.origin->value->getVec3 ();
  565. glm::vec3 scale = glm::vec3 (1.0f);
  566. float angle = 0.0f;
  567. if (object.is<Image> ()) {
  568. const auto* image = object.as<Image> ();
  569. scale = image->scale->value->getVec3 ();
  570. angle = image->angles->value->getVec3 ().z;
  571. // cropoffset is already baked into the object's origin, adding it again shifts the layer
  572. } else if (object.is<Text> ()) {
  573. const auto* text = object.as<Text> ();
  574. scale = text->scale->value->getVec3 ();
  575. } else {
  576. scale = object.groupScale->value->getVec3 ();
  577. angle = object.groupAngles->value->getVec3 ().z;
  578. }
  579. return { origin, scale, angle };
  580. }
  581. CImage::ResolvedTransform CImage::resolveTransform (const Object& object) const {
  582. constexpr int kMaxParentDepth = 32;
  583. // Walk up the parent chain leaf-first, bounded by kMaxParentDepth to guard
  584. // against cycles. chain[0] is the requested object; the last entry is the root.
  585. const Object* chain[kMaxParentDepth + 1];
  586. int count = 0;
  587. const Object* current = &object;
  588. chain[count++] = current;
  589. while (current->parent.has_value ()) {
  590. if (count > kMaxParentDepth) {
  591. sLog.error ("Parent transform chain is too deep; possible cycle at object id=", current->id);
  592. break;
  593. }
  594. const auto* parentObject = this->getScene ().getObject (current->parent.value ());
  595. if (parentObject == nullptr) {
  596. break;
  597. }
  598. current = &parentObject->getObject ();
  599. chain[count++] = current;
  600. }
  601. // Accumulate top-down: the root's local transform is already its resolved
  602. // transform, then fold each child onto its already-resolved parent.
  603. ResolvedTransform resolved = localTransform (*chain[count - 1]);
  604. float meshPivotAngle = 0.0f;
  605. for (int i = count - 2; i >= 0; --i) {
  606. ResolvedTransform local = localTransform (*chain[i]);
  607. // scene.json's "attachment" follows a named point on the direct parent's puppet rig (see
  608. // PuppetAttachmentPoint), not the parent's own origin. This mirrors the real engine's attachment
  609. // resolution (confirmed via disassembly of wallpaper64.exe's sub_140148A20, the function that
  610. // actually builds an object's world matrix): parentWorldMatrix * boneLocalMatrix, composed with NO
  611. // Y-axis sign flip anywhere in the chain - the real engine uses one consistent Y convention all the
  612. // way from JSON through every level of parent/child composition, flipping (if at all) exactly once,
  613. // at the very end in the camera projection.
  614. //
  615. // resolveTransform's own "origin" space already works this same unflipped way for ordinary
  616. // (non-attachment) children two lines below (`local.origin.y = anchorOrigin.y + offset.y`, no
  617. // negation) - it's only the FINAL CImage-constructor/updateScenePosition step that ever flips Y, to
  618. // go from this consistent origin-space into screen/pixel space. The bone's meshPosition, however,
  619. // comes from getAttachmentPointMeshTransform() already in that same unflipped origin-space
  620. // convention (see its own doc comment) - so it must be folded in raw, exactly like a normal child's
  621. // local.origin is, not re-flipped a second time. Confirmed against a real wallpaper with a genuinely
  622. // large bone rotation (mikasa/3764765600's "eye" attachment, ~-45 degrees): before this fix the
  623. // attachment landed off the top edge of the screen entirely; with position un-negated it lands
  624. // correctly on the face.
  625. //
  626. // anchorAngle (the bone's rotation, same sign/no-flip as position) rotates the attached child's own
  627. // local-origin nudge below, via the same offset-rotation every normal child already goes through -
  628. // that's required for *position* to track the bone correctly: a child's own declared origin is a
  629. // small offset in the attachment point's local frame, so it has to rotate along with whatever that
  630. // frame's current orientation is, same as it already scales along with the parent's current scale.
  631. // It also feeds the child's own final stored angle two lines below - the mathematically consistent
  632. // choice (attachmentWorldMatrix * childLocalMatrix), and the one actually confirmed working: mikasa's
  633. // eye (the only attachment point found so far riding a bone with genuine non-zero rotation) is
  634. // visible with this formula, just not at the correct angle (her declared local angle of ~44.6 degrees
  635. // and the eye bone's ~-45 degree rotation nearly cancel to ~0 net rotation, rendering as a thin
  636. // angular sliver instead of a natural lash contour - a real, unsolved cosmetic bug, tracked
  637. // separately, not this line).
  638. //
  639. // Two variants were tried and reverted, both regressions confirmed by the user on real hardware, not
  640. // just sandbox: (1) flipping only meshTransform->angle's sign within anchorAngle - since anchorAngle
  641. // also drives the offset-rotation above, this swung the eye's own (~355-unit) local-origin nudge by
  642. // nearly 90 degrees and pushed the object off the right edge of the screen entirely ("eyes completely
  643. // disappeared"). (2) splitting a separate finalAngle that dropped the bone's rotation from the final
  644. // angle entirely, reasoning that position and orientation could use different angles - this looked
  645. // like a plausible eyelash contour in an isolated sandbox crop, but the eye's own detail marks (a
  646. // small highlight dot, iris shading, a few lash strokes - confirmed via decode_tex.py on "mikasa
  647. // eye.tex": barely 0.7% of the canvas is non-transparent) are precisely positioned to overlay a
  648. // specific closed-eye crease baked into mikasaback's own texture; changing the mesh's rotation swings
  649. // those small marks to different screen pixels even though the object's own bounding-box center
  650. // doesn't move, and evidently rotated them off that tiny target entirely - user confirmed "eyes are
  651. // still invisible" with a real screenshot showing bare skin, no eye at all, where the sandbox crop had
  652. // suggested something was there. Reverted back to the single-anchorAngle formula below, which is the
  653. // last state confirmed actually visible (if wrongly rotated) on real hardware - a real fix for the
  654. // rotation needs to explain why a *different* angle would still hit the same crease, not just look
  655. // better in isolation.
  656. // meshPivotAngle: the remaining angle difference pivots around the mesh's own center, not the object origin
  657. glm::vec3 anchorOrigin = resolved.origin;
  658. float anchorAngle = resolved.angle;
  659. glm::vec2 anchorScale = { 1.0f, 1.0f };
  660. if (chain[i]->attachment.has_value () && chain[i]->parent.has_value ()) {
  661. const auto* parentCObject = this->getScene ().getObject (chain[i]->parent.value ());
  662. if (const auto* parentImage = dynamic_cast<const CImage*> (parentCObject); parentImage != nullptr) {
  663. if (const auto meshTransform = parentImage->getAttachmentPointMeshTransform (*chain[i]->attachment);
  664. meshTransform.has_value ()) {
  665. const glm::vec2 meshOffset = rotateVec2 (
  666. { meshTransform->position.x * resolved.scale.x, meshTransform->position.y * resolved.scale.y },
  667. resolved.angle
  668. );
  669. anchorOrigin.x = resolved.origin.x + meshOffset.x;
  670. anchorOrigin.y = resolved.origin.y + meshOffset.y;
  671. anchorAngle = resolved.angle + meshTransform->angle;
  672. // the bone's own scale (possibly negative, i.e. a mirrored bone) carries into whatever
  673. // rides it, same as position/rotation
  674. anchorScale = meshTransform->scale;
  675. // the attachment matrix carries a static rotation that only orients the point's own frame,
  676. // so it steers the child's offset but not its orientation, and is cancelled around the mesh center
  677. meshPivotAngle += -meshTransform->restAngle;
  678. if (!this->m_attachmentDiagnosticLogged.contains (chain[i]->id)) {
  679. this->m_attachmentDiagnosticLogged.insert (chain[i]->id);
  680. sLog.out (
  681. "Attachment resolve for ", chain[i]->name, " (", chain[i]->id, "): point=",
  682. *chain[i]->attachment, " meshPosition=(", meshTransform->position.x, ",",
  683. meshTransform->position.y, ") boneAngleDeg=", glm::degrees (meshTransform->angle),
  684. " boneScale=(", meshTransform->scale.x, ",", meshTransform->scale.y, ") parentOrigin=(",
  685. resolved.origin.x, ",", resolved.origin.y, ") parentScale=", resolved.scale.x,
  686. " anchorOrigin=(", anchorOrigin.x, ",", anchorOrigin.y, ") anchorAngleDeg=",
  687. glm::degrees (anchorAngle), " restAngleDeg=", glm::degrees (meshTransform->restAngle)
  688. );
  689. }
  690. }
  691. }
  692. }
  693. const glm::vec2 offset
  694. = rotateVec2 ({ local.origin.x * resolved.scale.x, local.origin.y * resolved.scale.y }, anchorAngle);
  695. local.origin.x = anchorOrigin.x + offset.x;
  696. local.origin.y = anchorOrigin.y + offset.y;
  697. local.origin.z = resolved.origin.z + local.origin.z * resolved.scale.z;
  698. local.scale.x *= anchorScale.x;
  699. local.scale.y *= anchorScale.y;
  700. resolved = { local.origin, local.scale * resolved.scale, local.angle + anchorAngle, meshPivotAngle };
  701. if (chain[i]->id == 134 && !this->m_finalOriginLogged.contains (chain[i]->id)) {
  702. this->m_finalOriginLogged.insert (chain[i]->id);
  703. sLog.out (
  704. "TEMP-DIAG final resolved origin for ", chain[i]->name, " (", chain[i]->id, "): anchorOrigin=(",
  705. anchorOrigin.x, ",", anchorOrigin.y, ") offset=(", offset.x, ",", offset.y, ") finalOrigin=(",
  706. resolved.origin.x, ",", resolved.origin.y, ")"
  707. );
  708. }
  709. }
  710. return resolved;
  711. }
  712. CImage::CImage (Wallpapers::CScene& scene, const Image& image) :
  713. CObject (scene, image), CRenderable (scene, image, *image.model->material), ScriptableObject (scene, image),
  714. m_sceneSpacePosition (GL_NONE), m_copySpacePosition (GL_NONE), m_passSpacePosition (GL_NONE),
  715. m_texcoordCopy (GL_NONE), m_texcoordPass (GL_NONE), m_modelViewProjectionScreen (),
  716. m_modelViewProjectionPass (glm::mat4 (1.0)), m_modelViewProjectionCopy (), m_modelViewProjectionScreenInverse (),
  717. m_modelViewProjectionPassInverse (glm::inverse (m_modelViewProjectionPass)), m_modelViewProjectionCopyInverse (),
  718. m_modelMatrix (), m_viewProjectionMatrix (), m_image (image), m_pos (), m_initialized (false) {
  719. this->registerProperty ("origin", *image.origin->value);
  720. this->registerProperty ("scale", *image.scale->value);
  721. this->registerProperty ("angles", *image.angles->value);
  722. this->registerProperty ("visible", *image.visible->value);
  723. this->registerProperty ("alpha", *image.alpha->value);
  724. this->registerProperty ("color", *image.color->value);
  725. this->registerProperty ("parallaxDepth", *image.parallaxDepth->value);
  726. this->registerEffectConstants (image.effects);
  727. auto scene_width = static_cast<float> (scene.getWidth ());
  728. auto scene_height = static_cast<float> (scene.getHeight ());
  729. const auto transform = this->resolveTransform (this->getImage ());
  730. glm::vec3 origin = transform.origin;
  731. glm::vec2 size = this->getSize ();
  732. glm::vec3 scale = transform.scale;
  733. this->detectTexture ();
  734. const bool placeholderTexture = this->m_texture == nullptr;
  735. if (this->m_texture == nullptr) {
  736. if (this->m_image.model->solidlayer && size.x == 0.0f && size.y == 0.0f) {
  737. size.x = static_cast<float> (scene.getCanvasWidth ());
  738. size.y = static_cast<float> (scene.getCanvasHeight ());
  739. }
  740. // TODO: create a dummy texture of correct size, fbo constructors should be enough, but this should be
  741. // properly handled
  742. // solid layers are often declared far larger than the scene, nothing samples
  743. // these buffers past the canvas so only the layout size has to stay as declared
  744. const glm::vec2 placeholderSize = { std::min (size.x, static_cast<float> (scene.getCanvasWidth ())),
  745. std::min (size.y, static_cast<float> (scene.getCanvasHeight ())) };
  746. this->m_texture = std::make_shared<CFBO> (
  747. "", TextureFormat_ARGB8888, TextureFlags_NoFlags, 1, size.x, size.y, placeholderSize.x, placeholderSize.y
  748. );
  749. }
  750. // If the wallpaper doesn't specify a size, fall back to the texture or model dimensions
  751. if ((size.x == 0.0f || size.y == 0.0f) && this->m_texture != nullptr) {
  752. size.x = static_cast<float> (this->m_texture->getRealWidth ());
  753. size.y = static_cast<float> (this->m_texture->getRealHeight ());
  754. } else if (
  755. (size.x == 0.0f || size.y == 0.0f) && this->getImage ().model->width.has_value ()
  756. && this->getImage ().model->height.has_value ()
  757. ) {
  758. size.x = static_cast<float> (this->getImage ().model->width.value ());
  759. size.y = static_cast<float> (this->getImage ().model->height.value ());
  760. }
  761. // fullscreen layers should use the whole projection's size
  762. // TODO: WHAT SHOULD AUTOSIZE DO?
  763. if (this->getImage ().model->fullscreen) {
  764. size = { static_cast<float> (scene.getCanvasWidth ()), static_cast<float> (scene.getCanvasHeight ()) };
  765. origin = { scene_width / 2, scene_height / 2, 0 };
  766. }
  767. this->m_size = size;
  768. // taken after the texture/model/fullscreen fallbacks above, unsized layers would otherwise get 0x0 buffers
  769. glm::vec2 bufferSize = size;
  770. if (placeholderTexture) {
  771. bufferSize = glm::min (bufferSize, glm::vec2 (scene.getCanvasWidth (), scene.getCanvasHeight ()));
  772. }
  773. this->updateScenePosition (origin, size, scale, scene_width, scene_height);
  774. // register both FBOs into the scene
  775. std::ostringstream nameA, nameB;
  776. // TODO: determine when _rt_imageLayerComposite and _rt_imageLayerAlbedo is used
  777. nameA << "_rt_imageLayerComposite_" << this->getImage ().id << "_a";
  778. nameB << "_rt_imageLayerComposite_" << this->getImage ().id << "_b";
  779. // scene.json's own "clampuvs" is a per-object override on top of whatever the base texture
  780. // asset defaults to - without it, effects that distort UVs near the edges (refraction, ripples)
  781. // can wrap around and sample the opposite edge of the buffer instead of clamping.
  782. // compose layers always clamp, their effects would otherwise wrap samples from the opposite edge
  783. const uint32_t compositeFlags = (this->getImage ().clampUVs || this->getImage ().model->passthrough)
  784. ? (this->m_texture->getFlags () | TextureFlags_ClampUVs)
  785. : this->m_texture->getFlags ();
  786. this->m_currentMainFBO = this->m_mainFBO = scene.create (
  787. nameA.str (), TextureFormat_ARGB8888, compositeFlags, 1, { bufferSize.x, bufferSize.y }, { bufferSize.x, bufferSize.y }
  788. );
  789. this->m_currentSubFBO = this->m_subFBO = scene.create (
  790. nameB.str (), TextureFormat_ARGB8888, compositeFlags, 1, { bufferSize.x, bufferSize.y }, { bufferSize.x, bufferSize.y }
  791. );
  792. GLfloat sceneSpacePosition[] = { this->m_pos.x, this->m_pos.y, 0.0f, this->m_pos.x, this->m_pos.w, 0.0f,
  793. this->m_pos.z, this->m_pos.y, 0.0f, this->m_pos.z, this->m_pos.y, 0.0f,
  794. this->m_pos.x, this->m_pos.w, 0.0f, this->m_pos.z, this->m_pos.w, 0.0f };
  795. float width = 1.0f;
  796. float height = 1.0f;
  797. if (this->getTexture ()->isAnimated ()) {
  798. // animated images use different coordinates as they're essentially a texture atlas
  799. width = static_cast<float> (this->getTexture ()->getRealWidth ())
  800. / static_cast<float> (this->getTexture ()->getTextureWidth (0));
  801. height = static_cast<float> (this->getTexture ()->getRealHeight ())
  802. / static_cast<float> (this->getTexture ()->getTextureHeight (0));
  803. }
  804. else if (
  805. this->getTexture () != nullptr
  806. && (this->getTexture ()->getTextureWidth (0) != this->getTexture ()->getRealWidth ()
  807. || this->getTexture ()->getTextureHeight (0) != this->getTexture ()->getRealHeight ())
  808. ) {
  809. // Account for padding in non-power-of-two textures: clamp UVs to the real content
  810. width = static_cast<float> (this->getTexture ()->getRealWidth ())
  811. / static_cast<float> (this->getTexture ()->getTextureWidth (0));
  812. height = static_cast<float> (this->getTexture ()->getRealHeight ())
  813. / static_cast<float> (this->getTexture ()->getTextureHeight (0));
  814. }
  815. // TODO: RECALCULATE THESE POSITIONS FOR PASSTHROUGH SO THEY TAKE THE RIGHT PART OF THE TEXTURE
  816. float x = 0.0f;
  817. float y = 0.0f;
  818. if (this->getTexture ()->isAnimated ()) {
  819. // animations should be copied completely
  820. x = 0.0f;
  821. y = 0.0f;
  822. width = 1.0f;
  823. height = 1.0f;
  824. }
  825. GLfloat realWidth = size.x;
  826. GLfloat realHeight = size.y;
  827. GLfloat realX = 0.0;
  828. GLfloat realY = 0.0;
  829. if (this->getImage ().model->passthrough) {
  830. // Passthrough shaders fill the destination FBO from texcoords and sample the scene using positions.
  831. // Keep the destination quad full-screen in local FBO space, but pass scene-space positions through.
  832. x = 0.0f;
  833. y = 0.0f;
  834. width = 1.0f;
  835. height = 1.0f;
  836. realX = this->m_pos.x;
  837. realY = this->m_pos.w;
  838. realWidth = this->m_pos.z;
  839. realHeight = this->m_pos.y;
  840. if (this->getImage ().model->fullscreen) {
  841. realX = -1.0;
  842. realY = -1.0;
  843. realWidth = 1.0;
  844. realHeight = 1.0;
  845. }
  846. }
  847. GLfloat texcoordCopy[] = { x, height, x, y, width, height, width, height, x, y, width, y };
  848. GLfloat copySpacePosition[] = { realX, realHeight, 0.0f, realX, realY, 0.0f, realWidth, realHeight, 0.0f,
  849. realWidth, realHeight, 0.0f, realX, realY, 0.0f, realWidth, realY, 0.0f };
  850. GLfloat texcoordPass[] = { 0.0f, 1.0f, 0.0f, 0.0f, 1.0f, 1.0f, 1.0f, 1.0f, 0.0f, 0.0f, 1.0f, 0.0f };
  851. GLfloat passSpacePosition[]
  852. = { -1.0, 1.0, 0.0f, -1.0, -1.0, 0.0f, 1.0, 1.0, 0.0f, 1.0, 1.0, 0.0f, -1.0, -1.0, 0.0f, 1.0, -1.0, 0.0f };
  853. glGenBuffers (1, &this->m_sceneSpacePosition);
  854. glBindBuffer (GL_ARRAY_BUFFER, this->m_sceneSpacePosition);
  855. glBufferData (GL_ARRAY_BUFFER, sizeof (sceneSpacePosition), sceneSpacePosition, GL_STATIC_DRAW);
  856. glGenBuffers (1, &this->m_copySpacePosition);
  857. glBindBuffer (GL_ARRAY_BUFFER, this->m_copySpacePosition);
  858. glBufferData (GL_ARRAY_BUFFER, sizeof (copySpacePosition), copySpacePosition, GL_STATIC_DRAW);
  859. glGenBuffers (1, &this->m_passSpacePosition);
  860. glBindBuffer (GL_ARRAY_BUFFER, this->m_passSpacePosition);
  861. glBufferData (GL_ARRAY_BUFFER, sizeof (passSpacePosition), passSpacePosition, GL_STATIC_DRAW);
  862. glGenBuffers (1, &this->m_texcoordCopy);
  863. glBindBuffer (GL_ARRAY_BUFFER, this->m_texcoordCopy);
  864. glBufferData (GL_ARRAY_BUFFER, sizeof (texcoordCopy), texcoordCopy, GL_STATIC_DRAW);
  865. glGenBuffers (1, &this->m_texcoordPass);
  866. glBindBuffer (GL_ARRAY_BUFFER, this->m_texcoordPass);
  867. glBufferData (GL_ARRAY_BUFFER, sizeof (texcoordPass), texcoordPass, GL_STATIC_DRAW);
  868. this->m_hasPuppetMesh = this->loadPuppetMesh (size);
  869. this->m_sceneCenter
  870. = glm::vec3 ((this->m_pos.x + this->m_pos.z) / 2.0f, (this->m_pos.y + this->m_pos.w) / 2.0f, 0.0f);
  871. this->m_modelViewProjectionScreen
  872. = this->getScene ().getCamera ().getProjection () * this->getScene ().getCamera ().getLookAt ();
  873. // must match m_modelViewProjectionScreen - updateScreenSpacePosition() may skip recomputing it
  874. this->m_modelViewProjectionScreenInverse = glm::inverse (this->m_modelViewProjectionScreen);
  875. this->updateEffectTextureProjection ();
  876. if (this->getImage ().model->passthrough) {
  877. this->m_modelViewProjectionCopy = this->m_modelViewProjectionScreen;
  878. } else {
  879. this->m_modelViewProjectionCopy = glm::ortho<float> (0.0, size.x, 0.0, size.y);
  880. }
  881. this->m_modelViewProjectionCopyInverse = glm::inverse (this->m_modelViewProjectionCopy);
  882. this->m_modelMatrix = glm::ortho<float> (0.0, size.x, 0.0, size.y);
  883. this->m_viewProjectionMatrix = glm::mat4 (1.0);
  884. // marks the texture as used, which starts video playback if it isn't already
  885. this->m_texture->incrementUsageCount ();
  886. }
  887. void CImage::updateTextures () const {
  888. this->getTexture ()->update ();
  889. for (const auto* pass : this->m_passes) {
  890. pass->updatePlaybackTextures ();
  891. }
  892. }
  893. bool CImage::containsScenePoint (const glm::vec2& point) const {
  894. // m_pos is stored centered on the scene with y pointing down, x/z are left/right and y/w bottom/top
  895. const float x = point.x - static_cast<float> (this->getScene ().getWidth ()) / 2.0f;
  896. const float y = static_cast<float> (this->getScene ().getHeight ()) / 2.0f - point.y;
  897. return x >= std::min (this->m_pos.x, this->m_pos.z) && x <= std::max (this->m_pos.x, this->m_pos.z)
  898. && y >= std::min (this->m_pos.y, this->m_pos.w) && y <= std::max (this->m_pos.y, this->m_pos.w);
  899. }
  900. glm::vec2 CImage::getSceneCenter () const {
  901. return { (this->m_pos.x + this->m_pos.z) / 2.0f + static_cast<float> (this->getScene ().getWidth ()) / 2.0f,
  902. static_cast<float> (this->getScene ().getHeight ()) / 2.0f - (this->m_pos.y + this->m_pos.w) / 2.0f };
  903. }
  904. void CImage::refreshScenePosition () {
  905. const auto sceneWidth = static_cast<float> (this->getScene ().getWidth ());
  906. const auto sceneHeight = static_cast<float> (this->getScene ().getHeight ());
  907. const auto transform = this->resolveTransform (this->getImage ());
  908. glm::vec3 origin = transform.origin;
  909. const glm::vec2 size = this->resolveGeometrySize (sceneWidth, sceneHeight, origin);
  910. this->updateScenePosition (origin, size, transform.scale, sceneWidth, sceneHeight);
  911. }
  912. CImage::~CImage () {
  913. this->m_texture->decrementUsageCount ();
  914. // delete passes first as they depend on the image's data
  915. for (auto* pass : this->m_allPasses.empty () ? this->m_passes : this->m_allPasses) {
  916. delete pass;
  917. }
  918. this->m_passes.clear ();
  919. this->m_allPasses.clear ();
  920. glDeleteBuffers (1, &this->m_sceneSpacePosition);
  921. glDeleteBuffers (1, &this->m_copySpacePosition);
  922. glDeleteBuffers (1, &this->m_passSpacePosition);
  923. glDeleteBuffers (1, &this->m_texcoordCopy);
  924. glDeleteBuffers (1, &this->m_texcoordPass);
  925. if (this->m_puppetSpacePosition != GL_NONE) {
  926. glDeleteBuffers (1, &this->m_puppetSpacePosition);
  927. }
  928. if (this->m_puppetTexCoord != GL_NONE) {
  929. glDeleteBuffers (1, &this->m_puppetTexCoord);
  930. }
  931. if (this->m_puppetIndices != GL_NONE) {
  932. glDeleteBuffers (1, &this->m_puppetIndices);
  933. }
  934. }
  935. bool CImage::loadPuppetMesh (const glm::vec2& size) {
  936. if (!this->getImage ().model->puppet.has_value ()) {
  937. return false;
  938. }
  939. try {
  940. const auto stream = this->getScene ().getScene ().project.assetLocator->read (*this->getImage ().model->puppet);
  941. std::vector<char> data { std::istreambuf_iterator<char> (*stream), std::istreambuf_iterator<char> () };
  942. constexpr size_t markerSize = 9;
  943. constexpr size_t meshHeaderSize = sizeof (uint32_t) * 2;
  944. const std::string puppetVersion
  945. = data.size () >= markerSize ? std::string (data.data (), strlen ("MDLV0021")) : "";
  946. const size_t mdlsOffset = [&data] () -> size_t {
  947. for (size_t offset = markerSize; offset + strlen ("MDLS") < data.size (); offset++) {
  948. if (std::memcmp (data.data () + offset, "MDLS", strlen ("MDLS")) == 0) {
  949. return offset;
  950. }
  951. }
  952. return data.size ();
  953. }();
  954. auto meshBuffer = std::make_unique<char[]> (data.size ());
  955. std::copy (data.begin (), data.end (), meshBuffer.get ());
  956. const BinaryReader reader (std::make_shared<MemoryStream> (std::move (meshBuffer), data.size ()));
  957. const bool isDiagTarget
  958. = this->getImage ().name == "bodyhairkochuru" || this->getImage ().name == "spiritblossomahribase";
  959. const auto layout = resolvePuppetVertexLayout (reader, markerSize, mdlsOffset, meshHeaderSize);
  960. if (!layout.has_value ()) {
  961. sLog.error ("Could not find a usable MDLV mesh block in ", *this->getImage ().model->puppet);
  962. return false;
  963. }
  964. const auto mesh = readPuppetMeshData (reader, layout->block, meshHeaderSize, layout->vertexStride);
  965. if (!mesh.has_value ()) {
  966. sLog.error ("Could not find a usable MDLV mesh block in ", *this->getImage ().model->puppet);
  967. return false;
  968. }
  969. this->m_puppetRawPositions = mesh->positions;
  970. this->updatePuppetPositionBuffer (size);
  971. glGenBuffers (1, &this->m_puppetTexCoord);
  972. glBindBuffer (GL_ARRAY_BUFFER, this->m_puppetTexCoord);
  973. glBufferData (GL_ARRAY_BUFFER, mesh->texcoords.size () * sizeof (GLfloat), mesh->texcoords.data (), GL_STATIC_DRAW);
  974. glGenBuffers (1, &this->m_puppetIndices);
  975. glBindBuffer (GL_ELEMENT_ARRAY_BUFFER, this->m_puppetIndices);
  976. glBufferData (
  977. GL_ELEMENT_ARRAY_BUFFER, mesh->indices.size () * sizeof (GLushort), mesh->indices.data (), GL_STATIC_DRAW
  978. );
  979. this->m_puppetIndexCount = static_cast<GLsizei> (mesh->indices.size ());
  980. if (isDiagTarget) {
  981. this->m_puppetTexCoordData = mesh->texcoords;
  982. this->m_puppetIndicesData = mesh->indices;
  983. }
  984. // TEMP-DIAG: per-triangle UV-vs-position area ratio
  985. if (isDiagTarget) {
  986. std::vector<double> ratios;
  987. ratios.reserve (mesh->indices.size () / 3);
  988. for (size_t t = 0; t + 2 < mesh->indices.size (); t += 3) {
  989. const auto i0 = mesh->indices[t], i1 = mesh->indices[t + 1], i2 = mesh->indices[t + 2];
  990. const glm::vec2 p0 (mesh->positions[i0 * 3], mesh->positions[i0 * 3 + 1]);
  991. const glm::vec2 p1 (mesh->positions[i1 * 3], mesh->positions[i1 * 3 + 1]);
  992. const glm::vec2 p2 (mesh->positions[i2 * 3], mesh->positions[i2 * 3 + 1]);
  993. const glm::vec2 u0 (mesh->texcoords[i0 * 2], mesh->texcoords[i0 * 2 + 1]);
  994. const glm::vec2 u1 (mesh->texcoords[i1 * 2], mesh->texcoords[i1 * 2 + 1]);
  995. const glm::vec2 u2 (mesh->texcoords[i2 * 2], mesh->texcoords[i2 * 2 + 1]);
  996. const double posArea = std::abs ((p1.x - p0.x) * (p2.y - p0.y) - (p2.x - p0.x) * (p1.y - p0.y));
  997. const double uvArea = std::abs ((u1.x - u0.x) * (u2.y - u0.y) - (u2.x - u0.x) * (u1.y - u0.y));
  998. if (posArea <= 1e-6) {
  999. continue;
  1000. }
  1001. ratios.push_back (uvArea / posArea);
  1002. }
  1003. std::vector<double> sorted = ratios;
  1004. std::sort (sorted.begin (), sorted.end ());
  1005. const double median = sorted.empty () ? 0.0 : sorted[sorted.size () / 2];
  1006. sLog.out (
  1007. "TEMP-DIAG uv/pos area ratio for ", this->getImage ().name, ": triCount=", ratios.size (), " median=", median
  1008. );
  1009. for (size_t t = 0; t + 2 < mesh->indices.size (); t += 3) {
  1010. const size_t triIndex = t / 3;
  1011. if (triIndex >= ratios.size ()) {
  1012. break;
  1013. }
  1014. const double ratio = ratios[triIndex];
  1015. if (median > 0.0 && (ratio > median * 20.0 || ratio < median / 20.0)) {
  1016. const auto i0 = mesh->indices[t], i1 = mesh->indices[t + 1], i2 = mesh->indices[t + 2];
  1017. sLog.out (
  1018. "TEMP-DIAG outlier tri=", triIndex, " ratio=", ratio, " verts=(", i0, ",", i1, ",", i2,
  1019. ") uv0=(", mesh->texcoords[i0 * 2], ",", mesh->texcoords[i0 * 2 + 1], ") uv1=(",
  1020. mesh->texcoords[i1 * 2], ",", mesh->texcoords[i1 * 2 + 1], ") uv2=(", mesh->texcoords[i2 * 2],
  1021. ",", mesh->texcoords[i2 * 2 + 1], ")"
  1022. );
  1023. }
  1024. }
  1025. }
  1026. sLog.out (
  1027. "Loaded puppet mesh ", *this->getImage ().model->puppet, " version=", puppetVersion, " stride=",
  1028. layout->vertexStride, " vertices=", this->m_puppetRawPositions.size () / 3, " indices=", this->m_puppetIndexCount
  1029. );
  1030. this->m_puppetBones.clear ();
  1031. this->m_puppetActiveAnimations.clear ();
  1032. this->m_puppetBlendIndices.clear ();
  1033. this->m_puppetBlendWeights.clear ();
  1034. this->m_puppetAttachmentPoints.clear ();
  1035. this->m_puppetBoneWorldAnimated.clear ();
  1036. const auto blend = readPuppetBlendData (reader, layout->block, meshHeaderSize, layout->vertexStride);
  1037. if (blend.has_value ()) {
  1038. this->m_puppetBlendIndices = blend->indices;
  1039. this->m_puppetBlendWeights = blend->weights;
  1040. }
  1041. if (mdlsOffset < data.size () && blend.has_value ()) {
  1042. try {
  1043. auto boneSet = parsePuppetBones (reader, mdlsOffset);
  1044. std::vector<int> bindParents (boneSet.bones.size ());
  1045. std::vector<glm::mat4> bindLocals (boneSet.bones.size ());
  1046. for (size_t i = 0; i < boneSet.bones.size (); i++) {
  1047. bindParents[i] = boneSet.bones[i].parent;
  1048. bindLocals[i] = boneSet.bones[i].bindLocal;
  1049. }
  1050. const std::vector<glm::mat4> worldBind = composeBoneWorldTransforms (bindParents, bindLocals);
  1051. for (size_t i = 0; i < boneSet.bones.size (); i++) {
  1052. boneSet.bones[i].inverseBindWorld = glm::inverse (worldBind[i]);
  1053. }
  1054. this->m_puppetBones = std::move (boneSet.bones);
  1055. this->m_puppetBoneWorldAnimated = worldBind;
  1056. // the MDLS "next section" field is trusted at face value below, but that's only been
  1057. // confirmed against MDLV0023 puppet-warp samples - other MDLV sub-formats (e.g. rope/particle
  1058. // rigs) may lay out MDLS differently, in which case this field is meaningless. It can point
  1059. // to either an optional MDAT (attachment points) section or straight to MDLA; cross-check
  1060. // which one (if either) it actually is before trusting anything read from that offset.
  1061. constexpr std::array<char, 4> mdlaMagic = { 'M', 'D', 'L', 'A' };
  1062. constexpr std::array<char, 4> mdatMagic = { 'M', 'D', 'A', 'T' };
  1063. const auto magicAt = [&data] (size_t offset, const std::array<char, 4>& magic) {
  1064. return offset + magic.size () <= data.size ()
  1065. && std::equal (magic.begin (), magic.end (), data.begin () + static_cast<long> (offset));
  1066. };
  1067. size_t mdlaOffset = boneSet.nextSectionOffset;
  1068. bool mdlaOffsetLooksValid = magicAt (mdlaOffset, mdlaMagic);
  1069. if (!mdlaOffsetLooksValid && magicAt (mdlaOffset, mdatMagic)) {
  1070. auto attachmentSet
  1071. = parsePuppetAttachmentPoints (reader, mdlaOffset, static_cast<uint32_t> (this->m_puppetBones.size ()));
  1072. this->m_puppetAttachmentPoints = std::move (attachmentSet.points);
  1073. mdlaOffset = attachmentSet.mdlaOffset;
  1074. mdlaOffsetLooksValid = magicAt (mdlaOffset, mdlaMagic);
  1075. }
  1076. std::vector<PuppetAnimationClip> clips;
  1077. if (mdlaOffsetLooksValid) {
  1078. clips = parsePuppetAnimationClips (
  1079. data, reader, mdlaOffset, static_cast<uint32_t> (this->m_puppetBones.size ()), isDiagTarget
  1080. );
  1081. } else {
  1082. sLog.error (
  1083. "Puppet MDLS data for ", *this->getImage ().model->puppet,
  1084. " doesn't lead to a recognizable MDLA section, skipping animation for this puppet"
  1085. );
  1086. }
  1087. // puppets can declare several simultaneous "additive" layers (idle sway, blinking, hand
  1088. // movement, ...) - collect every matching one here; updatePuppetSkinning blend-weights
  1089. // them together per bone using each layer's own "blend" setting.
  1090. for (const auto& layer : this->getImage ().animationLayers) {
  1091. auto match = std::find_if (clips.begin (), clips.end (), [&layer] (const PuppetAnimationClip& clip) {
  1092. return clip.name == layer->name;
  1093. });
  1094. if (match == clips.end ()) {
  1095. continue;
  1096. }
  1097. this->m_puppetActiveAnimations.push_back (
  1098. PuppetActiveAnimation { .clip = std::move (*match), .layer = layer.get () }
  1099. );
  1100. }
  1101. if (this->m_puppetActiveAnimations.empty () && !clips.empty () && !this->getImage ().animationLayers.empty ()) {
  1102. sLog.out (
  1103. "No puppet animation clip name matched an animation layer for ", *this->getImage ().model->puppet,
  1104. ", defaulting to the first clip (", clips.front ().name, ")"
  1105. );
  1106. this->m_puppetActiveAnimations.push_back (
  1107. PuppetActiveAnimation {
  1108. .clip = std::move (clips.front ()), .layer = this->getImage ().animationLayers.front ().get () }
  1109. );
  1110. }
  1111. for (const auto& active : this->m_puppetActiveAnimations) {
  1112. sLog.out (
  1113. "Playing puppet animation ", active.clip.name, " (", active.clip.mode, ", ", active.clip.fps,
  1114. " fps, ", active.clip.frameCount, " frames) on ", *this->getImage ().model->puppet
  1115. );
  1116. }
  1117. if (!this->m_puppetAttachmentPoints.empty ()) {
  1118. std::string names;
  1119. for (const auto& point : this->m_puppetAttachmentPoints) {
  1120. names += (names.empty () ? "" : ", ") + point.name;
  1121. }
  1122. sLog.out (
  1123. "Found ", this->m_puppetAttachmentPoints.size (), " puppet attachment point(s) on ",
  1124. *this->getImage ().model->puppet, ": ", names
  1125. );
  1126. }
  1127. } catch (const std::exception& ex) {
  1128. sLog.error (
  1129. "Could not load puppet skeleton/animation from ", *this->getImage ().model->puppet, ": ", ex.what (),
  1130. " (falling back to the static bind pose)"
  1131. );
  1132. this->m_puppetBones.clear ();
  1133. this->m_puppetActiveAnimations.clear ();
  1134. this->m_puppetAttachmentPoints.clear ();
  1135. this->m_puppetBoneWorldAnimated.clear ();
  1136. }
  1137. }
  1138. return true;
  1139. } catch (const std::exception& ex) {
  1140. sLog.error ("Could not load puppet mesh ", *this->getImage ().model->puppet, ": ", ex.what ());
  1141. return false;
  1142. }
  1143. }
  1144. void CImage::updatePuppetPositionBuffer (const glm::vec2& size) {
  1145. // once an animation clip is driving the mesh, its skinned output replaces the static bind pose
  1146. // as the source of truth - the bind pose (m_puppetRawPositions) is kept around unchanged, since
  1147. // skinning is recomputed from it fresh every frame, not accumulated from the previous frame
  1148. const auto& source
  1149. = !this->m_puppetActiveAnimations.empty () && !this->m_puppetSkinnedPositions.empty () ? this->m_puppetSkinnedPositions : this->m_puppetRawPositions;
  1150. if (source.empty ()) {
  1151. return;
  1152. }
  1153. // A puppet with effects is multi-pass: its geometry pass renders into its own object-sized
  1154. // intermediate FBO (see setupPasses(), the "writesToTarget" branch) using the local-canvas
  1155. // m_modelViewProjectionCopy projection, and later passes composite that FBO's texture onto the
  1156. // scene the normal (non-puppet) way - that first pass still needs plain local canvas coordinates
  1157. // (0..size, matching its texcoords) to line up with that projection. Only a puppet with no
  1158. // effects skips straight from its one and only pass to the shared scene FBO, using
  1159. // m_modelViewProjectionScreen (see setupPasses()) - that path needs vertices already in the same
  1160. // absolute scene-space coordinates uploadGeometryBuffers() bakes into sceneSpacePosition for a
  1161. // normal quad, or every vertex renders shifted by a constant offset equal to wherever this object
  1162. // should have been, reading as the whole mesh floating somewhere else on screen entirely.
  1163. const bool bakeScenePosition = this->m_passes.size () <= 1 || this->m_puppetMeshLast;
  1164. std::vector<GLfloat> positions;
  1165. positions.reserve (source.size ());
  1166. for (size_t index = 0; index + 2 < source.size (); index += 3) {
  1167. const float localX = size.x / 2.0f + source[index];
  1168. const float localY = size.y / 2.0f - source[index + 1];
  1169. if (bakeScenePosition) {
  1170. // maps the local-canvas coordinate onto this object's scene-space bounding box; m_pos.w is
  1171. // its bottom edge (m_pos.y is the top, see updateScenePosition()) so localY==0 has to land
  1172. // there, not on m_pos.y, or the puppet renders vertically flipped
  1173. positions.push_back (this->m_pos.x + localX * this->m_puppetScale.x);
  1174. positions.push_back (this->m_pos.w + localY * this->m_puppetScale.y);
  1175. } else {
  1176. positions.push_back (localX);
  1177. positions.push_back (localY);
  1178. }
  1179. // raw .mdl Z values aren't used by this engine's orthographic puppet compositing (depth test
  1180. // is disabled for puppets; layering comes from draw order + alpha blending) - and glm::ortho's
  1181. // clip.z = -localZ has no near/far normalization, so a puppet's real mesh depth (tens of units)
  1182. // would get clipped outside [-1,1] and lose most of the mesh. Zero it instead.
  1183. positions.push_back (0.0f);
  1184. }
  1185. // skip the constructor's pre-setup() call, where m_passes/m_pos/m_puppetScale aren't resolved yet
  1186. if (!this->m_puppetPositionDiagnosticLogged && !this->m_passes.empty ()) {
  1187. this->m_puppetPositionDiagnosticLogged = true;
  1188. glm::vec3 boundsMin (std::numeric_limits<float>::max ());
  1189. glm::vec3 boundsMax (std::numeric_limits<float>::lowest ());
  1190. for (size_t i = 0; i + 2 < positions.size (); i += 3) {
  1191. const glm::vec3 p (positions[i], positions[i + 1], positions[i + 2]);
  1192. boundsMin = glm::min (boundsMin, p);
  1193. boundsMax = glm::max (boundsMax, p);
  1194. }
  1195. sLog.out (
  1196. "Puppet position bake for ", this->getImage ().name, " (", this->getId (), "): bakeScenePosition=",
  1197. bakeScenePosition, " passes=", this->m_passes.size (), " vertexCount=", positions.size () / 3,
  1198. " boundsMin=(", boundsMin.x, ",", boundsMin.y, ",", boundsMin.z, ") boundsMax=(", boundsMax.x, ",",
  1199. boundsMax.y, ",", boundsMax.z, ")"
  1200. );
  1201. }
  1202. if (this->m_puppetSpacePosition == GL_NONE) {
  1203. glGenBuffers (1, &this->m_puppetSpacePosition);
  1204. }
  1205. glBindBuffer (GL_ARRAY_BUFFER, this->m_puppetSpacePosition);
  1206. glBufferData (GL_ARRAY_BUFFER, positions.size () * sizeof (GLfloat), positions.data (), GL_DYNAMIC_DRAW);
  1207. }
  1208. namespace {
  1209. glm::vec3 lerp (const glm::vec3& a, const glm::vec3& b, float alpha) { return a + (b - a) * alpha; }
  1210. }
  1211. void CImage::updatePuppetSkinning () {
  1212. if (this->m_puppetActiveAnimations.empty () || this->m_puppetBones.empty ()) {
  1213. return;
  1214. }
  1215. if (this->getScene ().getContext ().getApp ().getContext ().settings.render.debug.noPuppetAnimation) {
  1216. return;
  1217. }
  1218. // every matching, currently-visible animation layer plays and blends by its own "blend" weight,
  1219. // instead of only the first one. bindLocal from the MDLS array is deliberately not used as a
  1220. // rotation baseline - a clip's own sample is used directly, since some files carry bones whose
  1221. // MDLS bindLocal translation is wildly different from what their animation samples say, and
  1222. // falling back to it visibly detaches whatever that bone drives.
  1223. struct ActiveLayerSample {
  1224. const PuppetAnimationClip* clip;
  1225. uint32_t frame0;
  1226. uint32_t frame1;
  1227. float alpha;
  1228. float blend;
  1229. };
  1230. std::vector<ActiveLayerSample> samples;
  1231. for (const auto& candidate : this->m_puppetActiveAnimations) {
  1232. if (candidate.layer == nullptr || !candidate.layer->visible->value->getBool ()) {
  1233. continue;
  1234. }
  1235. const auto& clip = candidate.clip;
  1236. const float duration = clip.fps > 0.0f ? static_cast<float> (clip.frameCount) / clip.fps : 0.0f;
  1237. const float rate = candidate.layer->rate->value->getFloat ();
  1238. float frameFloat = 0.0f;
  1239. if (duration > 0.0f) {
  1240. // "mirror" clips play forward then backward, so the end flows back into the start instead of snapping
  1241. const bool mirror = std::ranges::equal (clip.mode, std::string_view ("mirror"), [] (char a, char b) {
  1242. return std::tolower (static_cast<unsigned char> (a)) == b;
  1243. });
  1244. const float period = mirror ? duration * 2.0f : duration;
  1245. float elapsed = std::fmod (g_Time * rate, period);
  1246. if (elapsed < 0.0f) {
  1247. elapsed += period;
  1248. }
  1249. if (mirror && elapsed > duration) {
  1250. elapsed = period - elapsed;
  1251. }
  1252. frameFloat = elapsed * clip.fps;
  1253. }
  1254. const auto frame0 = std::min (static_cast<uint32_t> (frameFloat), clip.frameCount);
  1255. samples.push_back (ActiveLayerSample {
  1256. .clip = &clip, .frame0 = frame0, .frame1 = std::min (frame0 + 1, clip.frameCount),
  1257. .alpha = frameFloat - static_cast<float> (frame0), .blend = candidate.layer->blend->value->getFloat () });
  1258. }
  1259. if (samples.empty ()) {
  1260. return;
  1261. }
  1262. std::vector<int> animatedParents (this->m_puppetBones.size ());
  1263. std::vector<glm::mat4> animatedLocals (this->m_puppetBones.size ());
  1264. for (size_t i = 0; i < this->m_puppetBones.size (); i++) {
  1265. const auto& bone = this->m_puppetBones[i];
  1266. animatedParents[i] = bone.parent;
  1267. const glm::vec3 bindPosition (bone.bindLocal[3]);
  1268. glm::vec3 position = bindPosition;
  1269. bool positionBased = false;
  1270. glm::vec3 rotation (0.0f);
  1271. glm::vec3 scale (1.0f);
  1272. bool anyTrack = false;
  1273. // each layer contributes a blend-weighted delta from the shared baseline (bind position, zero
  1274. // rotation, unit scale) rather than replacing it outright
  1275. for (const auto& sample : samples) {
  1276. if (i >= sample.clip->boneTracks.size () || sample.clip->boneTracks[i].size () <= sample.frame1) {
  1277. continue;
  1278. }
  1279. anyTrack = true;
  1280. const auto& track = sample.clip->boneTracks[i];
  1281. const glm::vec3 trackPosition = lerp (track[sample.frame0].position, track[sample.frame1].position, sample.alpha);
  1282. const glm::vec3 trackRotation = lerp (track[sample.frame0].rotation, track[sample.frame1].rotation, sample.alpha);
  1283. const glm::vec3 trackScale = lerp (track[sample.frame0].scale, track[sample.frame1].scale, sample.alpha);
  1284. // deltas are measured from the clip's own first frame, some rigs carry a static track pose far from bindLocal
  1285. const glm::vec3 restPosition = track[0].position;
  1286. if (!positionBased) {
  1287. position = restPosition;
  1288. positionBased = true;
  1289. }
  1290. position += sample.blend * (trackPosition - restPosition);
  1291. rotation += sample.blend * trackRotation;
  1292. scale += sample.blend * (trackScale - glm::vec3 (1.0f));
  1293. }
  1294. glm::mat4 local = glm::translate (glm::mat4 (1.0f), position);
  1295. local = glm::rotate (local, rotation.z, glm::vec3 (0.0f, 0.0f, 1.0f));
  1296. local = glm::rotate (local, rotation.y, glm::vec3 (0.0f, 1.0f, 0.0f));
  1297. local = glm::rotate (local, rotation.x, glm::vec3 (1.0f, 0.0f, 0.0f));
  1298. local = glm::scale (local, scale);
  1299. animatedLocals[i] = local;
  1300. // TEMP-DIAG: bones 29/30 logged every frame
  1301. if ((i == 29 || i == 30) && this->getImage ().name == "bodyhairkochuru") {
  1302. sLog.out (
  1303. "TEMP-DIAG bone anim for ", this->getImage ().name, " (", this->getId (), ") i=", i, " parent=",
  1304. bone.parent, " bindLocalPos=(", bone.bindLocal[3].x, ",", bone.bindLocal[3].y, ") animatedPos=(",
  1305. position.x, ",", position.y, ",", position.z, ") rotationDeg=(", glm::degrees (rotation.x), ",",
  1306. glm::degrees (rotation.y), ",", glm::degrees (rotation.z), ") scale=(", scale.x, ",", scale.y, ",",
  1307. scale.z, ") hasTrack=", anyTrack, " activeLayers=", samples.size (), " time=", g_Time
  1308. );
  1309. }
  1310. }
  1311. const std::vector<glm::mat4> worldAnimated = composeBoneWorldTransforms (animatedParents, animatedLocals);
  1312. // attachment points (see getAttachmentPointMeshTransform) need the live bone transforms independently
  1313. // of the skin matrices below, which fold in the inverse bind pose
  1314. this->m_puppetBoneWorldAnimated = worldAnimated;
  1315. std::vector<glm::mat4> skinMatrices (this->m_puppetBones.size ());
  1316. for (size_t i = 0; i < this->m_puppetBones.size (); i++) {
  1317. skinMatrices[i] = worldAnimated[i] * this->m_puppetBones[i].inverseBindWorld;
  1318. }
  1319. const size_t vertexCount = this->m_puppetRawPositions.size () / 3;
  1320. this->m_puppetSkinnedPositions.assign (this->m_puppetRawPositions.size (), 0.0f);
  1321. for (size_t v = 0; v < vertexCount; v++) {
  1322. const glm::vec4 bindPos (
  1323. this->m_puppetRawPositions[v * 3], this->m_puppetRawPositions[v * 3 + 1], this->m_puppetRawPositions[v * 3 + 2],
  1324. 1.0f
  1325. );
  1326. glm::vec3 skinned (0.0f);
  1327. const glm::uvec4& indices = v < this->m_puppetBlendIndices.size () ? this->m_puppetBlendIndices[v] : glm::uvec4 (0);
  1328. const glm::vec4& weights = v < this->m_puppetBlendWeights.size () ? this->m_puppetBlendWeights[v] : glm::vec4 (0.0f);
  1329. for (int influence = 0; influence < 4; influence++) {
  1330. const float weight = weights[influence];
  1331. if (weight == 0.0f) {
  1332. continue;
  1333. }
  1334. const uint32_t boneIndex = indices[influence];
  1335. if (boneIndex >= skinMatrices.size ()) {
  1336. continue;
  1337. }
  1338. skinned += weight * glm::vec3 (skinMatrices[boneIndex] * bindPos);
  1339. }
  1340. this->m_puppetSkinnedPositions[v * 3] = skinned.x;
  1341. this->m_puppetSkinnedPositions[v * 3 + 1] = skinned.y;
  1342. this->m_puppetSkinnedPositions[v * 3 + 2] = skinned.z;
  1343. }
  1344. // TEMP-DIAG: triangles that overlap on screen post-skinning despite sampling distant UV regions
  1345. if (!this->m_puppetOverlapDiagLogged && !this->m_puppetIndicesData.empty ()
  1346. && (this->getImage ().name == "bodyhairkochuru" || this->getImage ().name == "spiritblossomahribase")) {
  1347. this->m_puppetOverlapDiagLogged = true;
  1348. struct TriBounds {
  1349. glm::vec2 min, max, uvCentroid;
  1350. };
  1351. std::vector<TriBounds> tris;
  1352. const size_t triCount = this->m_puppetIndicesData.size () / 3;
  1353. tris.reserve (triCount);
  1354. for (size_t t = 0; t < triCount; t++) {
  1355. const auto i0 = this->m_puppetIndicesData[t * 3];
  1356. const auto i1 = this->m_puppetIndicesData[t * 3 + 1];
  1357. const auto i2 = this->m_puppetIndicesData[t * 3 + 2];
  1358. const glm::vec2 p0 (this->m_puppetSkinnedPositions[i0 * 3], this->m_puppetSkinnedPositions[i0 * 3 + 1]);
  1359. const glm::vec2 p1 (this->m_puppetSkinnedPositions[i1 * 3], this->m_puppetSkinnedPositions[i1 * 3 + 1]);
  1360. const glm::vec2 p2 (this->m_puppetSkinnedPositions[i2 * 3], this->m_puppetSkinnedPositions[i2 * 3 + 1]);
  1361. const glm::vec2 uv0 (this->m_puppetTexCoordData[i0 * 2], this->m_puppetTexCoordData[i0 * 2 + 1]);
  1362. const glm::vec2 uv1 (this->m_puppetTexCoordData[i1 * 2], this->m_puppetTexCoordData[i1 * 2 + 1]);
  1363. const glm::vec2 uv2 (this->m_puppetTexCoordData[i2 * 2], this->m_puppetTexCoordData[i2 * 2 + 1]);
  1364. tris.push_back (TriBounds {
  1365. .min = glm::min (p0, glm::min (p1, p2)), .max = glm::max (p0, glm::max (p1, p2)),
  1366. .uvCentroid = (uv0 + uv1 + uv2) / 3.0f });
  1367. }
  1368. size_t overlapCount = 0;
  1369. for (size_t a = 0; a < triCount && overlapCount < 15; a++) {
  1370. for (size_t b = a + 1; b < triCount && overlapCount < 15; b++) {
  1371. const auto& ta = tris[a];
  1372. const auto& tb = tris[b];
  1373. const bool boxesOverlap
  1374. = ta.min.x <= tb.max.x && ta.max.x >= tb.min.x && ta.min.y <= tb.max.y && ta.max.y >= tb.min.y;
  1375. if (!boxesOverlap) {
  1376. continue;
  1377. }
  1378. if (glm::distance (ta.uvCentroid, tb.uvCentroid) < 0.15f) {
  1379. continue;
  1380. }
  1381. overlapCount++;
  1382. sLog.out (
  1383. "TEMP-DIAG overlap for ", this->getImage ().name, " tri", a, " box=(", ta.min.x, ",", ta.min.y, ")-(",
  1384. ta.max.x, ",", ta.max.y, ") uv=(", ta.uvCentroid.x, ",", ta.uvCentroid.y, ") vs tri", b, " box=(",
  1385. tb.min.x, ",", tb.min.y, ")-(", tb.max.x, ",", tb.max.y, ") uv=(", tb.uvCentroid.x, ",", tb.uvCentroid.y,
  1386. ")"
  1387. );
  1388. }
  1389. }
  1390. sLog.out (
  1391. "TEMP-DIAG overlap scan for ", this->getImage ().name, " done: triCount=", triCount, " overlapsLogged=",
  1392. overlapCount
  1393. );
  1394. }
  1395. this->updatePuppetPositionBuffer (this->m_size);
  1396. }
  1397. std::optional<CImage::AttachmentPointTransform> CImage::getAttachmentPointMeshTransform (const std::string& name) const {
  1398. if (this->m_puppetBoneWorldAnimated.empty ()) {
  1399. return std::nullopt;
  1400. }
  1401. const auto it = std::find_if (
  1402. this->m_puppetAttachmentPoints.begin (), this->m_puppetAttachmentPoints.end (),
  1403. [&name] (const PuppetAttachmentPoint& point) { return point.name == name; }
  1404. );
  1405. if (it == this->m_puppetAttachmentPoints.end () || static_cast<size_t> (it->boneIndex) >= this->m_puppetBoneWorldAnimated.size ()) {
  1406. return std::nullopt;
  1407. }
  1408. const glm::mat4 animatedWorld = this->m_puppetBoneWorldAnimated[it->boneIndex] * it->localTransform;
  1409. const float angle = std::atan2 (animatedWorld[0][1], animatedWorld[0][0]);
  1410. // scale.y = det(X,Y)/scale.x, projecting the transformed Y-basis onto what an unreflected
  1411. // rotation by `angle` would have produced - comes out negative if the bone's matrix includes a
  1412. // reflection (mirrored bone), instead of folding that into a bogus rotation angle
  1413. const float scaleX = glm::length (glm::vec2 (animatedWorld[0]));
  1414. const glm::vec2 scale
  1415. = scaleX > 1e-6f ? glm::vec2 (
  1416. scaleX, (animatedWorld[0][0] * animatedWorld[1][1] - animatedWorld[0][1] * animatedWorld[1][0]) / scaleX
  1417. )
  1418. : glm::vec2 (scaleX, glm::length (glm::vec2 (animatedWorld[1])));
  1419. const glm::mat4 bindWorld = glm::inverse (this->m_puppetBones[it->boneIndex].inverseBindWorld) * it->localTransform;
  1420. const float restAngle = std::atan2 (bindWorld[0][1], bindWorld[0][0]);
  1421. return AttachmentPointTransform {
  1422. .position = glm::vec3 (animatedWorld[3]), .angle = angle, .scale = scale, .restAngle = restAngle
  1423. };
  1424. }
  1425. void CImage::setupPuppetGeometryCallback (Effects::CPass* pass) const {
  1426. pass->setGeometryCallback (
  1427. [this, pass] () {
  1428. const GLint position = glGetAttribLocation (pass->getProgramID (), "a_Position");
  1429. const GLint texCoord = glGetAttribLocation (pass->getProgramID (), "a_TexCoord");
  1430. if (!this->m_puppetDrawDiagnosticLogged) {
  1431. this->m_puppetDrawDiagnosticLogged = true;
  1432. sLog.out (
  1433. "Puppet draw setup for ", this->getImage ().name, " (", this->getId (), "): programID=",
  1434. pass->getProgramID (), " a_Position=", position, " a_TexCoord=", texCoord, " indexCount=",
  1435. this->m_puppetIndexCount, " size=", this->m_size.x, "x", this->m_size.y
  1436. );
  1437. }
  1438. if (position >= 0) {
  1439. glEnableVertexAttribArray (position);
  1440. glBindBuffer (GL_ARRAY_BUFFER, this->m_puppetSpacePosition);
  1441. glVertexAttribPointer (position, 3, GL_FLOAT, GL_FALSE, 0, nullptr);
  1442. }
  1443. if (texCoord >= 0) {
  1444. glEnableVertexAttribArray (texCoord);
  1445. glBindBuffer (GL_ARRAY_BUFFER, this->m_puppetTexCoord);
  1446. glVertexAttribPointer (texCoord, 2, GL_FLOAT, GL_FALSE, 0, nullptr);
  1447. }
  1448. // updatePuppetPositionBuffer flips Y when converting mesh-space positions to screen space,
  1449. // which mirrors the mesh and reverses triangle winding relative to what the MDL file's index
  1450. // buffer encodes - but not necessarily uniformly across the whole mesh, since real puppet
  1451. // meshes aren't guaranteed to be consistently wound to begin with. Puppet content is flat 2D
  1452. // art with no real backface concept, so rather than chase the "correct" winding, just never
  1453. // cull it - a material requesting cullmode "normal" would otherwise silently drop whichever
  1454. // subset of triangles ends up on the wrong side, which looks like patchy missing geometry.
  1455. glDisable (GL_CULL_FACE);
  1456. },
  1457. [this, pass] () {
  1458. GLint currentFramebuffer = 0;
  1459. glGetIntegerv (GL_DRAW_FRAMEBUFFER_BINDING, &currentFramebuffer);
  1460. if (currentFramebuffer != static_cast<GLint> (this->getScene ().getFBO ()->getFramebuffer ())) {
  1461. GLfloat previousClearColor[4] = {};
  1462. glGetFloatv (GL_COLOR_CLEAR_VALUE, previousClearColor);
  1463. glClearColor (0.0f, 0.0f, 0.0f, 0.0f);
  1464. glClear (GL_COLOR_BUFFER_BIT);
  1465. glClearColor (
  1466. previousClearColor[0], previousClearColor[1], previousClearColor[2], previousClearColor[3]
  1467. );
  1468. }
  1469. glBindBuffer (GL_ELEMENT_ARRAY_BUFFER, this->m_puppetIndices);
  1470. glDrawElements (GL_TRIANGLES, this->m_puppetIndexCount, GL_UNSIGNED_SHORT, nullptr);
  1471. {
  1472. static int mikasaEyeDumpCounter = 0;
  1473. if (this->getId () == 603 && mikasaEyeDumpCounter++ == 5) {
  1474. GLint vp[4] = {};
  1475. glGetIntegerv (GL_VIEWPORT, vp);
  1476. const int w = vp[2], h = vp[3];
  1477. if (w > 0 && h > 0 && w < 8192 && h < 8192) {
  1478. std::vector<unsigned char> pixels (static_cast<size_t> (w) * h * 4);
  1479. glReadPixels (0, 0, w, h, GL_RGBA, GL_UNSIGNED_BYTE, pixels.data ());
  1480. FILE* f = fopen ("/tmp/mikasa_eye_bakepass_dump.raw", "wb");
  1481. if (f) {
  1482. fwrite (&w, sizeof (int), 1, f);
  1483. fwrite (&h, sizeof (int), 1, f);
  1484. fwrite (pixels.data (), 1, pixels.size (), f);
  1485. fclose (f);
  1486. sLog.out ("TEMP-DIAG dumped FBO contents for mikasa eye bake pass: ", w, "x", h, " to /tmp/mikasa_eye_bakepass_dump.raw");
  1487. }
  1488. }
  1489. }
  1490. }
  1491. if (!this->m_puppetDrawErrorChecked) {
  1492. this->m_puppetDrawErrorChecked = true;
  1493. GLint boundFBO = 0;
  1494. GLint viewport[4] = {};
  1495. GLint boundTexture = 0;
  1496. glGetIntegerv (GL_DRAW_FRAMEBUFFER_BINDING, &boundFBO);
  1497. glGetIntegerv (GL_VIEWPORT, viewport);
  1498. glActiveTexture (GL_TEXTURE0);
  1499. glGetIntegerv (GL_TEXTURE_BINDING_2D, &boundTexture);
  1500. const GLenum err = glGetError ();
  1501. const GLboolean cullEnabled = glIsEnabled (GL_CULL_FACE);
  1502. const GLboolean depthEnabled = glIsEnabled (GL_DEPTH_TEST);
  1503. const GLboolean scissorEnabled = glIsEnabled (GL_SCISSOR_TEST);
  1504. const GLboolean blendEnabled = glIsEnabled (GL_BLEND);
  1505. GLint cullFaceMode = 0, frontFace = 0;
  1506. glGetIntegerv (GL_CULL_FACE_MODE, &cullFaceMode);
  1507. glGetIntegerv (GL_FRONT_FACE, &frontFace);
  1508. GLboolean colorMask[4] = {};
  1509. glGetBooleanv (GL_COLOR_WRITEMASK, colorMask);
  1510. sLog.out (
  1511. "Puppet draw result for ", this->getImage ().name, " (", this->getId (), "): glError=", err,
  1512. " boundFBO=", boundFBO, " sceneFBO=", this->getScene ().getFBO ()->getFramebuffer (), " viewport=(",
  1513. viewport[0], ",", viewport[1], ",", viewport[2], ",", viewport[3], ") boundTexture=", boundTexture,
  1514. " ownTextureReady=", (this->getTexture () != nullptr && this->getTexture ()->isReady ()), " ownTextureID=",
  1515. (this->getTexture () != nullptr ? this->getTexture ()->getTextureID (0) : 0), " color4=(",
  1516. this->getColor4 ().r, ",", this->getColor4 ().g, ",", this->getColor4 ().b, ",", this->getColor4 ().a,
  1517. ") alpha=", this->getUserAlpha (), " brightness=", this->getBrightness (), " cullEnabled=",
  1518. (int) cullEnabled, " cullFaceMode=", cullFaceMode, " frontFace=", frontFace, " depthEnabled=",
  1519. (int) depthEnabled, " scissorEnabled=", (int) scissorEnabled, " blendEnabled=", (int) blendEnabled,
  1520. " colorMask=(", (int) colorMask[0], ",", (int) colorMask[1], ",", (int) colorMask[2], ",",
  1521. (int) colorMask[3], ")"
  1522. );
  1523. }
  1524. },
  1525. [pass] () {
  1526. const GLint position = glGetAttribLocation (pass->getProgramID (), "a_Position");
  1527. const GLint texCoord = glGetAttribLocation (pass->getProgramID (), "a_TexCoord");
  1528. if (position >= 0) {
  1529. glDisableVertexAttribArray (position);
  1530. }
  1531. if (texCoord >= 0) {
  1532. glDisableVertexAttribArray (texCoord);
  1533. }
  1534. }
  1535. );
  1536. }
  1537. void CImage::addEffectPasses (const ImageEffect& effect) {
  1538. const auto fboProvider = std::make_shared<FBOProvider> (this);
  1539. for (const auto& fbo : effect.effect->fbos) {
  1540. fboProvider->create (
  1541. *fbo,
  1542. this->m_image.model->passthrough ? (this->m_texture->getFlags () | TextureFlags_ClampUVs)
  1543. : this->m_texture->getFlags (),
  1544. this->getSize ()
  1545. );
  1546. }
  1547. auto curEffect = effect.effect->passes.begin ();
  1548. auto endEffect = effect.effect->passes.end ();
  1549. auto curOverride = effect.passOverrides.begin ();
  1550. auto endOverride = effect.passOverrides.end ();
  1551. for (; curEffect != endEffect; ++curEffect) {
  1552. if (!(*curEffect)->material.has_value ()) {
  1553. if (!(*curEffect)->command.has_value ()) {
  1554. sLog.error ("Pass without material and command not supported");
  1555. continue;
  1556. }
  1557. if (!(*curEffect)->source.has_value ()) {
  1558. sLog.error ("Pass without material and source not supported");
  1559. continue;
  1560. }
  1561. if (!(*curEffect)->target.has_value ()) {
  1562. sLog.error ("Pass without material and target not supported");
  1563. continue;
  1564. }
  1565. if ((*curEffect)->command != Command_Copy) {
  1566. sLog.error ("Only copy command is supported for pass without material");
  1567. continue;
  1568. }
  1569. auto virtualPass
  1570. = std::make_unique<MaterialPass> (MaterialPass { .blending = BlendingMode_Normal,
  1571. .cullmode = CullingMode_Disable,
  1572. .depthtest = DepthtestMode_Disabled,
  1573. .depthwrite = DepthwriteMode_Disabled,
  1574. .shader = "commands/copy",
  1575. .textures = { { 0, *(*curEffect)->source } },
  1576. .combos = {},
  1577. .constants = {} });
  1578. const auto& config = *this->m_virtualPassess.emplace_back (std::move (virtualPass));
  1579. this->m_passes.push_back (new CPass (
  1580. *this, fboProvider, config, std::nullopt, std::nullopt, (*curEffect)->target.value ()
  1581. ));
  1582. } else {
  1583. for (auto& pass : (*curEffect)->material.value ()->passes) {
  1584. const auto override = curOverride != endOverride
  1585. ? **curOverride
  1586. : std::optional<std::reference_wrapper<const ImageEffectPassOverride>> (std::nullopt);
  1587. const auto target = (*curEffect)->target.has_value ()
  1588. ? *(*curEffect)->target
  1589. : std::optional<std::reference_wrapper<std::string>> (std::nullopt);
  1590. this->m_passes.push_back (
  1591. new CPass (*this, fboProvider, *pass, override, (*curEffect)->binds, target)
  1592. );
  1593. }
  1594. if (curOverride != endOverride) {
  1595. ++curOverride;
  1596. }
  1597. }
  1598. }
  1599. }
  1600. void CImage::setup () {
  1601. if (this->m_initialized) {
  1602. return;
  1603. }
  1604. // TODO: SUPPORT PASSTHROUGH (IT'S A SHADER)
  1605. // passthrough without effects has nothing to draw
  1606. if (this->m_image.model->passthrough && this->m_image.effects.empty ()) {
  1607. return;
  1608. }
  1609. const auto& debug = this->getScene ().getContext ().getApp ().getContext ().settings.render.debug;
  1610. for (const auto& cur : this->getImage ().model->material->passes) {
  1611. this->m_passes.push_back (
  1612. new CPass (*this, std::make_shared<FBOProvider> (this), *cur, std::nullopt, std::nullopt, std::nullopt)
  1613. );
  1614. }
  1615. std::vector<const DynamicValue*> passVisibility (this->m_passes.size (), nullptr);
  1616. std::vector<bool> passFromEffect (this->m_passes.size (), false);
  1617. if (!debug.baseOnly && !this->getImage ().effects.empty ()) {
  1618. for (const auto& cur : this->m_image.effects) {
  1619. if (std::find (debug.skipEffects.begin (), debug.skipEffects.end (), static_cast<int> (cur->id))
  1620. != debug.skipEffects.end ()) {
  1621. continue;
  1622. }
  1623. const auto effectVisibility = this->getScene ().getContext ().getApp ().getContext ().resolveEffectVisibility (
  1624. static_cast<int> (cur->id), cur->name
  1625. );
  1626. // an explicit --disable-effect/--enable-effect override wins over the scene's own visibility
  1627. if (effectVisibility.has_value () && !*effectVisibility) {
  1628. continue;
  1629. }
  1630. // scripts can toggle hidden effects at runtime; puppets can't, their mesh pass layout
  1631. // depends on the pass count
  1632. const bool followsVisibility = !effectVisibility.has_value () && !this->m_hasPuppetMesh;
  1633. if (!followsVisibility && !effectVisibility.has_value () && !cur->visible->value->getBool ()) {
  1634. continue;
  1635. }
  1636. const DynamicValue* visibleValue = followsVisibility ? cur->visible->value.get () : nullptr;
  1637. const size_t firstEffectPass = this->m_passes.size ();
  1638. try {
  1639. this->addEffectPasses (*cur);
  1640. } catch (const std::exception& e) {
  1641. if (visibleValue == nullptr || visibleValue->getBool ()) {
  1642. throw;
  1643. }
  1644. for (size_t i = firstEffectPass; i < this->m_passes.size (); i++) {
  1645. delete this->m_passes[i];
  1646. }
  1647. this->m_passes.resize (firstEffectPass);
  1648. sLog.error (
  1649. "Dropping hidden effect ", cur->id, " (", cur->name, ") on ", this->getImage ().name, ": ",
  1650. e.what ()
  1651. );
  1652. continue;
  1653. }
  1654. passVisibility.resize (this->m_passes.size (), visibleValue);
  1655. passFromEffect.resize (this->m_passes.size (), true);
  1656. }
  1657. }
  1658. const size_t passCountBeforeTrailingPasses = this->m_passes.size ();
  1659. if (!debug.baseOnly) {
  1660. const auto magentaCompositeTint = findMagentaCompositeTint (this->m_image, debug.skipEffects);
  1661. if (magentaCompositeTint.has_value ()) {
  1662. auto tintOverride = std::make_unique<ImageEffectPassOverride> (ImageEffectPassOverride {
  1663. .id = -1,
  1664. .combos = {
  1665. { "BLENDMODE", 30 },
  1666. },
  1667. .constants = {},
  1668. .textures = {},
  1669. });
  1670. tintOverride->constants.emplace ("color", UserSettingBuilder::fromValue (magentaCompositeTint.value ()));
  1671. tintOverride->constants.emplace ("alpha", UserSettingBuilder::fromValue (1.0f));
  1672. this->m_materials.compatibilityMaterials.emplace_back (
  1673. MaterialParser::load (this->getScene ().getScene ().project, "materials/effects/tint.json")
  1674. );
  1675. this->m_materials.compatibilityOverrides.emplace_back (std::move (tintOverride));
  1676. this->m_passes.push_back (new CPass (
  1677. *this, std::make_shared<FBOProvider> (this),
  1678. **this->m_materials.compatibilityMaterials.back ()->passes.begin (),
  1679. *this->m_materials.compatibilityOverrides.back (), std::nullopt, std::nullopt
  1680. ));
  1681. }
  1682. }
  1683. const int colorBlendMode = this->m_image.colorBlendMode->value->getInt ();
  1684. const bool readByOtherLayer = std::ranges::any_of (this->getScene ().getScene ().objects, [this] (const auto& object) {
  1685. return object->id != this->getImage ().id
  1686. && std::ranges::find (object->dependencies, this->getImage ().id) != object->dependencies.end ();
  1687. });
  1688. // WE keeps the result of a layer another one reads in _a and only copies it to the screen from there,
  1689. // drawing the last effect pass straight to the screen would leave _a one pass behind (or empty)
  1690. const bool copyForReaders = readByOtherLayer && this->getImage ().visible->value->getBool ();
  1691. if (!debug.baseOnly && (colorBlendMode > 0 || copyForReaders)) {
  1692. this->m_materials.colorBlending.material
  1693. = MaterialParser::load (this->getScene ().getScene ().project, "materials/util/effectpassthrough.json");
  1694. this->m_materials.colorBlending.override = std::make_unique<ImageEffectPassOverride> (ImageEffectPassOverride {
  1695. .id = -1,
  1696. .combos = colorBlendMode > 0 ? ComboMap { { "BLENDMODE", colorBlendMode } } : ComboMap {},
  1697. .constants = {},
  1698. .textures = {},
  1699. });
  1700. this->m_passes.push_back (new CPass (
  1701. *this, std::make_shared<FBOProvider> (this), **this->m_materials.colorBlending.material->passes.begin (),
  1702. *this->m_materials.colorBlending.override, std::nullopt, std::nullopt
  1703. ));
  1704. }
  1705. if (this->m_hasPuppetMesh && !this->m_passes.empty ()) {
  1706. this->m_puppetMeshPass = this->m_passes.front ();
  1707. this->m_puppetMeshLast = this->m_passes.size () == 1;
  1708. // effect masks are laid out over the source texture, so effects run on the flat texture first
  1709. // and the warped mesh is drawn last, sampling their output
  1710. const auto& materialPasses = this->getImage ().model->material->passes;
  1711. const bool hasTrailingPasses = this->m_passes.size () != passCountBeforeTrailingPasses;
  1712. if (this->m_passes.size () > 1 && !hasTrailingPasses && materialPasses.size () == 1
  1713. && materialPasses.front ()->constants.empty ()) {
  1714. const auto& base = *materialPasses.front ();
  1715. const auto& config = *this->m_virtualPassess.emplace_back (std::make_unique<MaterialPass> (MaterialPass {
  1716. .blending = base.blending,
  1717. .cullmode = base.cullmode,
  1718. .depthtest = base.depthtest,
  1719. .depthwrite = base.depthwrite,
  1720. .shader = base.shader,
  1721. .textures = {},
  1722. .usertextures = {},
  1723. .combos = base.combos,
  1724. .constants = {},
  1725. }));
  1726. this->m_puppetMeshPass
  1727. = new CPass (*this, std::make_shared<FBOProvider> (this), config, std::nullopt, std::nullopt, std::nullopt);
  1728. this->m_passes.push_back (this->m_puppetMeshPass);
  1729. this->m_puppetMeshLast = true;
  1730. }
  1731. }
  1732. passVisibility.resize (this->m_passes.size (), nullptr);
  1733. passFromEffect.resize (this->m_passes.size (), false);
  1734. for (size_t i = 0; i < this->m_passes.size (); i++) {
  1735. this->m_allPassStates.push_back (
  1736. { passVisibility[i], this->m_passes[i]->getBlendingMode (), passFromEffect[i] }
  1737. );
  1738. }
  1739. this->m_allPasses = this->m_passes;
  1740. CRenderable::setup ();
  1741. this->rebuildActivePasses ();
  1742. this->m_initialized = true;
  1743. }
  1744. bool CImage::effectVisibilityChanged () const {
  1745. for (size_t i = 0; i < this->m_allPassStates.size (); i++) {
  1746. const auto* visible = this->m_allPassStates[i].visible;
  1747. if (visible != nullptr && visible->getBool () != this->m_activePassMask[i]) {
  1748. return true;
  1749. }
  1750. }
  1751. return false;
  1752. }
  1753. void CImage::rebuildActivePasses () {
  1754. this->m_passes.clear ();
  1755. this->m_activePassMask.assign (this->m_allPasses.size (), false);
  1756. this->m_hasActiveEffectPass = false;
  1757. for (size_t i = 0; i < this->m_allPasses.size (); i++) {
  1758. const auto& state = this->m_allPassStates[i];
  1759. this->m_allPasses[i]->setBlendingMode (state.blending);
  1760. if (state.visible == nullptr || state.visible->getBool ()) {
  1761. this->m_activePassMask[i] = true;
  1762. this->m_hasActiveEffectPass |= state.fromEffect;
  1763. this->m_passes.push_back (this->m_allPasses[i]);
  1764. }
  1765. }
  1766. // if there's more than one pass the blendmode has to be moved from the beginning to the end
  1767. if (this->m_passes.size () > 1) {
  1768. const auto first = this->m_passes.begin ();
  1769. const auto last = this->m_passes.rbegin ();
  1770. (*last)->setBlendingMode ((*first)->getBlendingMode ());
  1771. (*first)->setBlendingMode (BlendingMode_Normal);
  1772. }
  1773. // setupPasses() ping-pongs these, every rebuild has to start from the same pair
  1774. this->m_currentMainFBO = this->m_mainFBO;
  1775. this->m_currentSubFBO = this->m_subFBO;
  1776. this->setupPasses ();
  1777. }
  1778. void CImage::setupPasses () {
  1779. // like WE, start on whichever buffer makes the last offscreen pass land in _a, which is what other layers read
  1780. auto offscreenPasses = std::ranges::count_if (this->m_passes, [] (const Effects::CPass* pass) {
  1781. return !pass->getTarget ().has_value ();
  1782. });
  1783. if (!this->m_passes.empty () && !this->m_passes.back ()->getTarget ().has_value ()
  1784. && this->shouldRenderFinalPass (true)) {
  1785. offscreenPasses--;
  1786. }
  1787. if (offscreenPasses % 2 == 0) {
  1788. std::swap (this->m_currentMainFBO, this->m_currentSubFBO);
  1789. }
  1790. std::shared_ptr<const CFBO> drawTo = this->m_currentMainFBO;
  1791. std::shared_ptr<const TextureProvider> asInput = this->getTexture ();
  1792. GLuint texcoord = this->getTexCoordCopy ();
  1793. auto cur = this->m_passes.begin ();
  1794. auto end = this->m_passes.end ();
  1795. bool first = true;
  1796. bool inTargetEffectSequence = false;
  1797. std::shared_ptr<const TextureProvider> effectInput = nullptr;
  1798. for (; cur != end; ++cur) {
  1799. Effects::CPass* pass = *cur;
  1800. std::shared_ptr<const CFBO> prevDrawTo = drawTo;
  1801. bool writesToTarget = false;
  1802. const bool isFirstPass = first;
  1803. const bool isMeshPass = this->m_hasPuppetMesh && pass == this->m_puppetMeshPass;
  1804. GLuint spacePosition = isMeshPass ? this->m_puppetSpacePosition
  1805. : isFirstPass ? this->getCopySpacePosition ()
  1806. : this->getPassSpacePosition ();
  1807. const glm::mat4* projection
  1808. = (isFirstPass) ? &this->m_modelViewProjectionCopy : &this->m_modelViewProjectionPass;
  1809. const glm::mat4* inverseProjection
  1810. = (isFirstPass) ? &this->m_modelViewProjectionCopyInverse : &this->m_modelViewProjectionPassInverse;
  1811. first = false;
  1812. if (isMeshPass) {
  1813. pass->setBlendingMode (BlendingMode_Translucent);
  1814. this->setupPuppetGeometryCallback (pass);
  1815. }
  1816. pass->setModelMatrix (&this->m_modelMatrix);
  1817. pass->setViewProjectionMatrix (&this->m_viewProjectionMatrix);
  1818. pass->setEffectTextureProjectionMatrix (&this->m_effectTextureProjection, &this->m_effectTextureProjectionInverse);
  1819. writesToTarget = this->configurePassTarget (pass, drawTo, asInput, effectInput, inTargetEffectSequence);
  1820. // TODO: PROPERLY CHECK IF THIS IS ALL THAT'S NEEDED
  1821. if (!writesToTarget && this->shouldRenderFinalPass (std::next (cur) == end)) {
  1822. drawTo = this->getScene ().getFBO ();
  1823. // A puppet with no effects has its geometry pass be both the first AND the last pass, drawn
  1824. // straight into the shared scene FBO below - same as any other object's final pass, so it
  1825. // needs the same screen-space projection. updatePuppetPositionBuffer() bakes this object's
  1826. // resolved scene position/scale directly into m_puppetSpacePosition (mirroring what
  1827. // uploadGeometryBuffers does for a normal quad's sceneSpacePosition), so m_modelViewProjectionScreen
  1828. // is the correct projection for those vertices now, not the local-canvas m_modelViewProjectionCopy
  1829. // this pass otherwise uses when rendering to an intermediate, object-sized target. The
  1830. // spacePosition reassignment below is a no-op for puppets either way - the puppet geometry
  1831. // callback always binds m_puppetSpacePosition itself, ignoring whatever spacePosition holds.
  1832. spacePosition = this->getSceneSpacePosition ();
  1833. projection = &this->m_modelViewProjectionScreen;
  1834. // WE's final pass inverse lands in the layer's local space (origin at its center, unscaled
  1835. // pixels); older shaders like the bundled xray.vert unproject the pointer through it
  1836. inverseProjection = &this->m_objectSpaceProjectionInverse;
  1837. }
  1838. pass->setDestination (drawTo);
  1839. pass->setInput (asInput);
  1840. pass->setPreviousInput (inTargetEffectSequence ? effectInput : nullptr);
  1841. pass->setPosition (spacePosition);
  1842. pass->setTexCoord (texcoord);
  1843. pass->setModelViewProjectionMatrix (projection);
  1844. pass->setModelViewProjectionMatrixInverse (inverseProjection);
  1845. texcoord = this->getTexCoordPass ();
  1846. if (writesToTarget) {
  1847. asInput = drawTo;
  1848. drawTo = prevDrawTo;
  1849. } else {
  1850. drawTo = prevDrawTo;
  1851. this->pinpongFramebuffer (&drawTo, &asInput);
  1852. inTargetEffectSequence = false;
  1853. effectInput = nullptr;
  1854. }
  1855. }
  1856. }
  1857. bool CImage::shouldRenderFinalPass (bool isLastPass) const {
  1858. const auto& appContext = this->getScene ().getContext ().getApp ().getContext ();
  1859. const auto visibility = appContext.resolveObjectVisibility (this->getId (), this->getObject ().name);
  1860. const bool visible = visibility.value_or (this->getImage ().visible->value->getBool ());
  1861. if (!isLastPass || !visible) {
  1862. return false;
  1863. }
  1864. const auto& debug = this->getScene ().getContext ().getApp ().getContext ().settings.render.debug;
  1865. return !(debug.noSolidFinal && this->getImage ().model->solidlayer);
  1866. }
  1867. bool CImage::configurePassTarget (
  1868. Effects::CPass* pass, std::shared_ptr<const CFBO>& drawTo, const std::shared_ptr<const TextureProvider>& asInput,
  1869. std::shared_ptr<const TextureProvider>& effectInput, bool& inTargetEffectSequence
  1870. ) {
  1871. if (!pass->getTarget ().has_value ()) {
  1872. return false;
  1873. }
  1874. const std::string target = pass->getTarget ().value ();
  1875. std::shared_ptr<const CFBO> resolved = pass->getFBOProvider ()->find (target);
  1876. if (resolved == nullptr) {
  1877. resolved = this->getScene ().findFBO (target);
  1878. }
  1879. if (resolved == nullptr) {
  1880. sLog.error (
  1881. "Pass target FBO '", target, "' could not be resolved for object ", pass->getRenderable ().getId (),
  1882. " shader=", pass->getPass ().shader
  1883. );
  1884. return false;
  1885. }
  1886. if (!inTargetEffectSequence) {
  1887. effectInput = asInput;
  1888. inTargetEffectSequence = true;
  1889. }
  1890. drawTo = resolved;
  1891. return true;
  1892. }
  1893. void CImage::pinpongFramebuffer (std::shared_ptr<const CFBO>* drawTo, std::shared_ptr<const TextureProvider>* asInput) {
  1894. std::shared_ptr<const CFBO> currentMainFBO = this->m_currentMainFBO;
  1895. std::shared_ptr<const CFBO> currentSubFBO = this->m_currentSubFBO;
  1896. if (drawTo != nullptr) {
  1897. *drawTo = currentSubFBO;
  1898. }
  1899. if (asInput != nullptr) {
  1900. *asInput = currentMainFBO;
  1901. }
  1902. this->m_currentMainFBO = currentSubFBO;
  1903. this->m_currentSubFBO = currentMainFBO;
  1904. }
  1905. void CImage::render () {
  1906. if (!this->m_initialized) {
  1907. return;
  1908. }
  1909. const auto& appContext = this->getScene ().getContext ().getApp ().getContext ();
  1910. const auto visibility = appContext.resolveObjectVisibility (this->getId (), this->getObject ().name);
  1911. // a hidden layer another object reads through _rt_imageLayerComposite_<id> (xray's "bloody" twins) still
  1912. // has to fill that FBO every frame, shouldRenderFinalPass() keeps it off the screen
  1913. if (!visibility.value_or (this->getImage ().visible->value->getBool ())
  1914. && (visibility.has_value () || !this->m_isDependency)) {
  1915. return;
  1916. }
  1917. if (this->effectVisibilityChanged ()) {
  1918. this->rebuildActivePasses ();
  1919. }
  1920. if (this->m_image.model->passthrough && !this->m_hasActiveEffectPass) {
  1921. return;
  1922. }
  1923. glColorMask (true, true, true, true);
  1924. this->updateScreenSpacePosition ();
  1925. if (this->m_hasPuppetMesh) {
  1926. this->updatePuppetSkinning ();
  1927. }
  1928. #if !NDEBUG
  1929. std::string str = "Image ";
  1930. if (this->getScene ().getScene ().camera.bloom.enabled->value->getBool () && this->getId () == -1) {
  1931. str += "bloom";
  1932. } else {
  1933. str += this->getImage ().name + " (" + std::to_string (this->getId ()) + ", "
  1934. + this->getImage ().model->material->filename + ")";
  1935. }
  1936. glPushDebugGroup (GL_DEBUG_SOURCE_APPLICATION, 0, -1, str.c_str ());
  1937. #endif /* DEBUG */
  1938. auto cur = this->m_passes.begin ();
  1939. const auto end = this->m_passes.end ();
  1940. for (; cur != end; ++cur) {
  1941. if (std::next (cur) == end) {
  1942. glColorMask (true, true, true, false);
  1943. }
  1944. (*cur)->render ();
  1945. }
  1946. // restore alpha writes - CParticle::render() never resets glColorMask, so leaving this
  1947. // disabled here leaks into the next frame's clear if bloom renders last
  1948. glColorMask (true, true, true, true);
  1949. #if !NDEBUG
  1950. glPopDebugGroup ();
  1951. #endif /* DEBUG */
  1952. }
  1953. const float& CImage::getBrightness () const { return this->m_image.brightness->value->getFloat (); }
  1954. const float& CImage::getUserAlpha () const { return this->getAlpha (); }
  1955. const float& CImage::getAlpha () const {
  1956. // some scenes store out-of-range alpha (e.g. 222) - it feeds mix() in blend modes, so it must stay in 0..1
  1957. m_alphaCache = glm::clamp (this->m_image.alpha->value->getFloat (), 0.0f, 1.0f);
  1958. return m_alphaCache;
  1959. }
  1960. const glm::vec3& CImage::getColor () const { return this->m_image.color->value->getVec3 (); }
  1961. const glm::vec4& CImage::getColor4 () const {
  1962. // "version" 2 materials take color and alpha together through g_Color4
  1963. m_color4Cache = glm::vec4 (this->m_image.color->value->getVec3 (), this->getAlpha ());
  1964. return m_color4Cache;
  1965. }
  1966. const glm::vec3& CImage::getCompositeColor () const { return this->m_image.color->value->getVec3 (); }
  1967. glm::vec2 CImage::resolveGeometrySize (float sceneWidth, float sceneHeight, glm::vec3& origin) const {
  1968. glm::vec2 size = this->getSize ();
  1969. if ((size.x == 0.0f || size.y == 0.0f) && this->m_texture != nullptr) {
  1970. size.x = static_cast<float> (this->m_texture->getRealWidth ());
  1971. size.y = static_cast<float> (this->m_texture->getRealHeight ());
  1972. } else if (
  1973. (size.x == 0.0f || size.y == 0.0f) && this->getImage ().model->width.has_value ()
  1974. && this->getImage ().model->height.has_value ()
  1975. ) {
  1976. size.x = static_cast<float> (this->getImage ().model->width.value ());
  1977. size.y = static_cast<float> (this->getImage ().model->height.value ());
  1978. }
  1979. if (this->getImage ().model->fullscreen) {
  1980. size = { static_cast<float> (this->getScene ().getCanvasWidth ()),
  1981. static_cast<float> (this->getScene ().getCanvasHeight ()) };
  1982. origin = { sceneWidth / 2.0f, sceneHeight / 2.0f, 0.0f };
  1983. }
  1984. return size;
  1985. }
  1986. void CImage::updateScenePosition (
  1987. const glm::vec3& origin, const glm::vec2& size, const glm::vec3& scale, float sceneWidth, float sceneHeight
  1988. ) {
  1989. const glm::vec2 scaledSize = size * glm::vec2 (scale);
  1990. this->m_pos.x = origin.x - (scaledSize.x / 2.0f);
  1991. this->m_pos.w = origin.y + (scaledSize.y / 2.0f);
  1992. this->m_pos.z = origin.x + (scaledSize.x / 2.0f);
  1993. this->m_pos.y = origin.y - (scaledSize.y / 2.0f);
  1994. const uint32_t alignment = this->getImage ().alignment;
  1995. if (alignment & ImageAlignment_Top) {
  1996. this->m_pos.y -= scaledSize.y / 2.0f;
  1997. this->m_pos.w -= scaledSize.y / 2.0f;
  1998. } else if (alignment & ImageAlignment_Bottom) {
  1999. this->m_pos.y += scaledSize.y / 2.0f;
  2000. this->m_pos.w += scaledSize.y / 2.0f;
  2001. }
  2002. if (alignment & ImageAlignment_Left) {
  2003. this->m_pos.x += scaledSize.x / 2.0f;
  2004. this->m_pos.z += scaledSize.x / 2.0f;
  2005. } else if (alignment & ImageAlignment_Right) {
  2006. this->m_pos.x -= scaledSize.x / 2.0f;
  2007. this->m_pos.z -= scaledSize.x / 2.0f;
  2008. }
  2009. this->m_pos.x -= sceneWidth / 2.0f;
  2010. this->m_pos.y = sceneHeight / 2.0f - this->m_pos.y;
  2011. this->m_pos.z -= sceneWidth / 2.0f;
  2012. this->m_pos.w = sceneHeight / 2.0f - this->m_pos.w;
  2013. }
  2014. void CImage::uploadGeometryBuffers (const glm::vec2& size) {
  2015. GLfloat sceneSpacePosition[] = { this->m_pos.x, this->m_pos.y, 0.0f, this->m_pos.x, this->m_pos.w, 0.0f,
  2016. this->m_pos.z, this->m_pos.y, 0.0f, this->m_pos.z, this->m_pos.y, 0.0f,
  2017. this->m_pos.x, this->m_pos.w, 0.0f, this->m_pos.z, this->m_pos.w, 0.0f };
  2018. float width = 1.0f;
  2019. float height = 1.0f;
  2020. if (this->getTexture () != nullptr && !this->getTexture ()->isAnimated ()
  2021. && (this->getTexture ()->getTextureWidth (0) != this->getTexture ()->getRealWidth ()
  2022. || this->getTexture ()->getTextureHeight (0) != this->getTexture ()->getRealHeight ())) {
  2023. width = static_cast<float> (this->getTexture ()->getRealWidth ())
  2024. / static_cast<float> (this->getTexture ()->getTextureWidth (0));
  2025. height = static_cast<float> (this->getTexture ()->getRealHeight ())
  2026. / static_cast<float> (this->getTexture ()->getTextureHeight (0));
  2027. }
  2028. float x = 0.0f;
  2029. float y = 0.0f;
  2030. GLfloat realWidth = size.x;
  2031. GLfloat realHeight = size.y;
  2032. GLfloat realX = 0.0f;
  2033. GLfloat realY = 0.0f;
  2034. if (this->getImage ().model->passthrough) {
  2035. width = 1.0f;
  2036. height = 1.0f;
  2037. realX = this->m_pos.x;
  2038. realY = this->m_pos.w;
  2039. realWidth = this->m_pos.z;
  2040. realHeight = this->m_pos.y;
  2041. if (this->getImage ().model->fullscreen) {
  2042. realX = -1.0f;
  2043. realY = -1.0f;
  2044. realWidth = 1.0f;
  2045. realHeight = 1.0f;
  2046. }
  2047. }
  2048. GLfloat texcoordCopy[] = { x, height, x, y, width, height, width, height, x, y, width, y };
  2049. GLfloat copySpacePosition[] = { realX, realHeight, 0.0f, realX, realY, 0.0f, realWidth, realHeight, 0.0f,
  2050. realWidth, realHeight, 0.0f, realX, realY, 0.0f, realWidth, realY, 0.0f };
  2051. glBindBuffer (GL_ARRAY_BUFFER, this->m_sceneSpacePosition);
  2052. glBufferData (GL_ARRAY_BUFFER, sizeof (sceneSpacePosition), sceneSpacePosition, GL_DYNAMIC_DRAW);
  2053. glBindBuffer (GL_ARRAY_BUFFER, this->m_copySpacePosition);
  2054. glBufferData (GL_ARRAY_BUFFER, sizeof (copySpacePosition), copySpacePosition, GL_DYNAMIC_DRAW);
  2055. glBindBuffer (GL_ARRAY_BUFFER, this->m_texcoordCopy);
  2056. glBufferData (GL_ARRAY_BUFFER, sizeof (texcoordCopy), texcoordCopy, GL_DYNAMIC_DRAW);
  2057. this->m_sceneCenter
  2058. = glm::vec3 ((this->m_pos.x + this->m_pos.z) / 2.0f, (this->m_pos.y + this->m_pos.w) / 2.0f, 0.0f);
  2059. this->m_modelViewProjectionCopy = this->getImage ().model->passthrough
  2060. ? this->m_modelViewProjectionScreen
  2061. : glm::ortho<float> (0.0, size.x, 0.0, size.y);
  2062. this->m_modelViewProjectionCopyInverse = glm::inverse (this->m_modelViewProjectionCopy);
  2063. this->m_modelMatrix = glm::ortho<float> (0.0, size.x, 0.0, size.y);
  2064. }
  2065. CImage::ResolvedTransform CImage::updateGeometryBuffers () {
  2066. auto sceneWidth = static_cast<float> (this->getScene ().getWidth ());
  2067. auto sceneHeight = static_cast<float> (this->getScene ().getHeight ());
  2068. const auto transform = this->resolveTransform (this->getImage ());
  2069. glm::vec3 origin = transform.origin;
  2070. const glm::vec3 scale = transform.scale;
  2071. const glm::vec2 size = this->resolveGeometrySize (sceneWidth, sceneHeight, origin);
  2072. this->m_size = size;
  2073. this->m_puppetScale = scale;
  2074. // must run before the puppet position buffer rebake below - it needs this frame's m_pos, not the
  2075. // previous one, to place puppet vertices at this object's actual scene position instead of its
  2076. // position from before whatever moved it (parallax, a script, an attachment point it follows, ...)
  2077. this->updateScenePosition (origin, size, scale, sceneWidth, sceneHeight);
  2078. if (this->m_pos != this->m_lastUploadedPos || size != this->m_lastUploadedGeometrySize) {
  2079. this->uploadGeometryBuffers (size);
  2080. // puppet vertices bake m_pos/scale in directly (see updatePuppetPositionBuffer), so they need
  2081. // the same "position or size changed" rebake trigger as the quad buffers above - a puppet whose
  2082. // animation is disabled (or one with no MDLA data at all, i.e. always static) would otherwise
  2083. // never get repositioned after its very first, load-time bake
  2084. if (this->m_hasPuppetMesh) {
  2085. this->updatePuppetPositionBuffer (size);
  2086. }
  2087. this->m_lastUploadedPos = this->m_pos;
  2088. this->m_lastUploadedGeometrySize = size;
  2089. }
  2090. return transform;
  2091. }
  2092. namespace {
  2093. // keeps an edge pair (e.g. m_pos.x/.z) from sliding past the viewport once `offset` is added to both,
  2094. // so the image never uncovers ground it doesn't have pixels for; an image too small to cover the viewport
  2095. // on this axis has no ground to uncover, it is an object sitting on the scene and moves freely
  2096. float clampParallaxAxis (float offset, float edgeA, float edgeB, float sceneExtent) {
  2097. const float low = std::min (edgeA, edgeB);
  2098. const float high = std::max (edgeA, edgeB);
  2099. const float half = sceneExtent / 2.0f;
  2100. const float maxOffset = -half - low;
  2101. const float minOffset = half - high;
  2102. if (minOffset > maxOffset)
  2103. return offset;
  2104. return std::clamp (offset, minOffset, maxOffset);
  2105. }
  2106. } // namespace
  2107. void CImage::updateScreenSpacePosition () {
  2108. const ResolvedTransform transform = this->updateGeometryBuffers ();
  2109. // angles are already in radians from scene.json; negated to account for the Y-flipped coordinate
  2110. // system (see CParticle.cpp)
  2111. const float angle = transform.angle;
  2112. glm::mat4 rotModel = glm::mat4 (1.0f);
  2113. if (angle != 0.0f) {
  2114. rotModel = glm::translate (rotModel, this->m_sceneCenter);
  2115. rotModel = glm::rotate (rotModel, -angle, glm::vec3 (0.0f, 0.0f, 1.0f));
  2116. rotModel = glm::translate (rotModel, -this->m_sceneCenter);
  2117. }
  2118. if (transform.meshPivotAngle != 0.0f && this->m_hasPuppetMesh) {
  2119. const auto& source = !this->m_puppetSkinnedPositions.empty () ? this->m_puppetSkinnedPositions : this->m_puppetRawPositions;
  2120. glm::vec2 boundsMin (std::numeric_limits<float>::max ());
  2121. glm::vec2 boundsMax (std::numeric_limits<float>::lowest ());
  2122. for (size_t i = 0; i + 2 < source.size (); i += 3) {
  2123. boundsMin = glm::min (boundsMin, glm::vec2 (source[i], source[i + 1]));
  2124. boundsMax = glm::max (boundsMax, glm::vec2 (source[i], source[i + 1]));
  2125. }
  2126. if (boundsMin.x <= boundsMax.x) {
  2127. const glm::vec2 meshCenter = (boundsMin + boundsMax) / 2.0f;
  2128. const glm::vec4 pivot (
  2129. this->m_pos.x + (this->m_size.x / 2.0f + meshCenter.x) * this->m_puppetScale.x,
  2130. this->m_pos.w + (this->m_size.y / 2.0f - meshCenter.y) * this->m_puppetScale.y, 0.0f, 1.0f
  2131. );
  2132. const glm::vec3 rotatedPivot = glm::vec3 (rotModel * pivot);
  2133. glm::mat4 pivotRot = glm::translate (glm::mat4 (1.0f), rotatedPivot);
  2134. pivotRot = glm::rotate (pivotRot, -transform.meshPivotAngle, glm::vec3 (0.0f, 0.0f, 1.0f));
  2135. pivotRot = glm::translate (pivotRot, -rotatedPivot);
  2136. rotModel = pivotRot * rotModel;
  2137. }
  2138. }
  2139. glm::mat4 mvp
  2140. = this->getScene ().getCamera ().getProjection () * this->getScene ().getCamera ().getLookAt () * rotModel;
  2141. // CScene::renderFrame() already folds disableparallax into getParallaxDisplacement()
  2142. if (this->getScene ().getScene ().camera.parallax.enabled->value->getBool ()) {
  2143. const glm::vec2 offset = this->getScene ().getParallaxOffset (this->getImage ());
  2144. float x = offset.x;
  2145. float y = offset.y;
  2146. // a texture that isn't UV-clamped tiles/repeats instead of showing black past its edges (GL_REPEAT,
  2147. // see CTexture.cpp), so sliding it further is harmless and exempt from the clamp; scene.json's own
  2148. // "clampuvs" overrides the base texture's flag the same way it does for the composite FBOs above
  2149. const bool textureTiles = !this->getImage ().clampUVs && this->getTexture () != nullptr
  2150. && (this->getTexture ()->getFlags () & TextureFlags_ClampUVs) == 0;
  2151. if (this->getScene ().getContext ().getApp ().getContext ().settings.mouse.clampParallaxToImageSize
  2152. && !textureTiles) {
  2153. const float sceneWidth = static_cast<float> (this->getScene ().getCanvasWidth ());
  2154. const float sceneHeight = static_cast<float> (this->getScene ().getCanvasHeight ());
  2155. x = clampParallaxAxis (x, this->m_pos.x, this->m_pos.z, sceneWidth);
  2156. y = clampParallaxAxis (y, this->m_pos.y, this->m_pos.w, sceneHeight);
  2157. }
  2158. mvp = glm::translate (mvp, { x, y, 0.0f });
  2159. }
  2160. // only the inverse is expensive; skip it when mvp didn't actually change
  2161. if (mvp != this->m_modelViewProjectionScreen) {
  2162. this->m_modelViewProjectionScreenInverse = glm::inverse (mvp);
  2163. }
  2164. this->m_modelViewProjectionScreen = mvp;
  2165. this->updateEffectTextureProjection ();
  2166. if (this->getImage ().model->passthrough) {
  2167. this->m_modelViewProjectionCopy = this->m_modelViewProjectionScreen;
  2168. this->m_modelViewProjectionCopyInverse = this->m_modelViewProjectionScreenInverse;
  2169. }
  2170. }
  2171. void CImage::updateEffectTextureProjection () {
  2172. // the final quad puts texcoord (0, 0) at (m_pos.x, m_pos.w), which is the layer's local (-1, +1) corner;
  2173. // the scene FBO is y-flipped against the screen the pointer position is measured on, hence the flip
  2174. const glm::vec3 center ((this->m_pos.x + this->m_pos.z) / 2.0f, (this->m_pos.y + this->m_pos.w) / 2.0f, 0.0f);
  2175. const glm::vec3 halfSize ((this->m_pos.z - this->m_pos.x) / 2.0f, (this->m_pos.w - this->m_pos.y) / 2.0f, 1.0f);
  2176. const glm::mat4 projection = glm::scale (glm::mat4 (1.0f), glm::vec3 (1.0f, -1.0f, 1.0f))
  2177. * this->m_modelViewProjectionScreen * glm::scale (glm::translate (glm::mat4 (1.0f), center), halfSize);
  2178. if (projection == this->m_effectTextureProjection) {
  2179. return;
  2180. }
  2181. this->m_effectTextureProjection = projection;
  2182. // a zero-sized layer has no inverse, keep the last usable one instead of feeding NaNs to the shader
  2183. if (halfSize.x != 0.0f && halfSize.y != 0.0f) {
  2184. this->m_effectTextureProjectionInverse = glm::inverse (projection);
  2185. }
  2186. const glm::vec2 size = this->getSize ();
  2187. this->m_objectSpaceProjectionInverse
  2188. = glm::scale (glm::mat4 (1.0f), glm::vec3 (size.x / 2.0f, size.y / 2.0f, 1.0f))
  2189. * this->m_effectTextureProjectionInverse;
  2190. }
  2191. const Image& CImage::getImage () const { return this->m_image; }
  2192. void CImage::markAsDependency () { this->m_isDependency = true; }
  2193. glm::vec2 CImage::getSize () const {
  2194. if (this->m_texture == nullptr) {
  2195. return this->getImage ().size;
  2196. }
  2197. // compose layers sample the whole scene, but effect masks map over the layer's own size
  2198. if (this->getImage ().model->passthrough && this->getImage ().size.x > 0.0f
  2199. && this->getImage ().size.y > 0.0f) {
  2200. return this->getImage ().size;
  2201. }
  2202. // solid layers use a stock white texture, the real footprint is declared by the scene
  2203. if (this->getImage ().model->solidlayer && this->getImage ().size.x > 0.0f && this->getImage ().size.y > 0.0f) {
  2204. return this->getImage ().size;
  2205. }
  2206. return { this->m_texture->getRealWidth (), this->m_texture->getRealHeight () };
  2207. }
  2208. GLuint CImage::getSceneSpacePosition () const { return this->m_sceneSpacePosition; }
  2209. GLuint CImage::getCopySpacePosition () const { return this->m_copySpacePosition; }
  2210. GLuint CImage::getPassSpacePosition () const { return this->m_passSpacePosition; }
  2211. GLuint CImage::getTexCoordCopy () const { return this->m_texcoordCopy; }
  2212. GLuint CImage::getTexCoordPass () const { return this->m_texcoordPass; }