Fix consistent crashing on 64 bit d3d

Cause: 64 bit pointer arithmetic. We assumed unsigned ints are 32 bits, while pointers are as well. In 64 bit, unsigned ints are still 32 bits but pointers are 64 bits. Converting between the two truncates half the data. This breaks pointers.

src/ImageCache.cpp

@@ -217,8 +217,8 @@ ImageCache::ReadFromDisk()
 
 struct ImageTexture : public RageTexture
 {
-	unsigned m_uTexHandle;
-	unsigned GetTexHandle() const override
+	intptr_t m_uTexHandle;
+	intptr_t GetTexHandle() const override
 	{
 		return m_uTexHandle;
 	}; // accessed by RageDisplay

src/RageBitmapTexture.h

@@ -13,15 +13,15 @@ class RageBitmapTexture : public RageTexture
 	/* only called by RageTextureManager::InvalidateTextures */
 	void Invalidate() override { m_uTexHandle = 0; /* don't Destroy() */ }
 	void Reload() override;
-	unsigned GetTexHandle() const override
+	intptr_t GetTexHandle() const override
 	{
 		return m_uTexHandle;
 	}; // accessed by RageDisplay
 
   private:
 	void Create(); // called by constructor and Reload
 	void Destroy();
-	unsigned m_uTexHandle; // treat as unsigned in OpenGL, ID3D8Texture* for D3D
+	intptr_t m_uTexHandle; // treat as unsigned in OpenGL, ID3D8Texture* for D3D
 };
 
 #endif

src/RageDisplay.h

@@ -229,23 +229,23 @@ class RageDisplay
 
 	/* return 0 if failed or internal texture resource handle
 	 * (unsigned in OpenGL, texture pointer in D3D) */
-	virtual unsigned CreateTexture(
+	virtual intptr_t CreateTexture(
 	  RagePixelFormat pixfmt, // format of img and of texture in video mem
 	  RageSurface* img,		  // must be in pixfmt
 	  bool bGenerateMipMaps) = 0;
-	virtual void UpdateTexture(unsigned iTexHandle,
+	virtual void UpdateTexture(intptr_t iTexHandle,
 							   RageSurface* img,
 							   int xoffset,
 							   int yoffset,
 							   int width,
 							   int height) = 0;
-	virtual void DeleteTexture(unsigned iTexHandle) = 0;
+	virtual void DeleteTexture(intptr_t iTexHandle) = 0;
 	/* Return an object to lock pixels for streaming. If not supported, returns
 	 * NULL. Delete the object normally. */
 	virtual RageTextureLock* CreateTextureLock() { return nullptr; }
 	virtual void ClearAllTextures() = 0;
 	virtual int GetNumTextureUnits() = 0;
-	virtual void SetTexture(TextureUnit, unsigned /* iTexture */) = 0;
+	virtual void SetTexture(TextureUnit, intptr_t /* iTexture */) = 0;
 	virtual void SetTextureMode(TextureUnit, TextureMode) = 0;
 	virtual void SetTextureWrapping(TextureUnit, bool) = 0;
 	virtual int GetMaxTextureSize() const = 0;
@@ -263,22 +263,22 @@ class RageDisplay
 	 * DeleteTexture. (UpdateTexture is not permitted.) Returns 0 if render-to-
 	 * texture is unsupported.
 	 */
-	virtual unsigned CreateRenderTarget(const RenderTargetParam&,
+	virtual intptr_t CreateRenderTarget(const RenderTargetParam&,
 										int& /* iTextureWidthOut */,
 										int& /* iTextureHeightOut */)
 	{
 		return 0;
 	}
 
-	virtual unsigned GetRenderTarget() { return 0; }
+	virtual intptr_t GetRenderTarget() { return 0; }
 
 	/* Set the render target, or 0 to resume rendering to the framebuffer. An
 	 * active render target may not be used as a texture. If bPreserveTexture is
 	 * true, the contents of the texture will be preserved from the previous
 	 * call; otherwise, cleared.  If bPreserveTexture is true the first time a
 	 * render target is used, behave as if bPreserveTexture was false.
 	 */
-	virtual void SetRenderTarget(unsigned /* iHandle */,
+	virtual void SetRenderTarget(intptr_t /* iHandle */,
 								 bool /* bPreserveTexture */ = true)
 	{
 	}
@@ -352,7 +352,7 @@ class RageDisplay
 	}
 	virtual RageSurface*
 	CreateScreenshot() = 0; // allocates a surface.  Caller must delete it.
-	virtual RageSurface* GetTexture(unsigned /* iTexture */)
+	virtual RageSurface* GetTexture(intptr_t /* iTexture */)
 	{
 		return nullptr;
 	} // allocates a surface.  Caller must delete it.

src/RageDisplay_D3D.cpp

@@ -57,21 +57,21 @@ IDirect3DSurface9* defaultDepthBuffer = 0;
 // of this application though.
 const D3DFORMAT g_DefaultAdapterFormat = D3DFMT_X8R8G8B8;
 
-static map<unsigned, RenderTarget*> g_mapRenderTargets;
+static map<intptr_t, RenderTarget*> g_mapRenderTargets;
 static RenderTarget* g_pCurrentRenderTarget = NULL;
 
 static bool g_bInvertY = false;
 
 /* Direct3D doesn't associate a palette with textures. Instead, we load a
  * palette into a slot. We need to keep track of which texture's palette is
  * stored in what slot. */
-map<unsigned, int> g_TexResourceToPaletteIndex;
+map<intptr_t, int> g_TexResourceToPaletteIndex;
 list<int> g_PaletteIndex;
 struct TexturePalette
 {
 	PALETTEENTRY p[256];
 };
-map<unsigned, TexturePalette> g_TexResourceToTexturePalette;
+map<intptr_t, TexturePalette> g_TexResourceToTexturePalette;
 
 // Load the palette, if any, for the given texture into a palette slot, and make
 // it current.
@@ -91,7 +91,7 @@ SetPalette(unsigned TexResource)
 
 		// If any other texture is currently using this slot, mark that palette
 		// unloaded.
-		for (map<unsigned, int>::iterator i =
+		for (map<intptr_t, int>::iterator i =
 			   g_TexResourceToPaletteIndex.begin();
 			 i != g_TexResourceToPaletteIndex.end();
 			 ++i) {
@@ -392,7 +392,7 @@ SetD3DParams(bool& bNewDeviceOut)
 	g_pd3dDevice->SetRenderState(D3DRS_NORMALIZENORMALS, TRUE);
 
 	// wipe old render targets
-	FOREACHM(unsigned, RenderTarget*, g_mapRenderTargets, rt)
+	FOREACHM(intptr_t, RenderTarget*, g_mapRenderTargets, rt)
 	delete rt->second;
 	g_mapRenderTargets.clear();
 
@@ -1211,7 +1211,7 @@ RageDisplay_D3D::GetNumTextureUnits()
 }
 
 void
-RageDisplay_D3D::SetTexture(TextureUnit tu, unsigned iTexture)
+RageDisplay_D3D::SetTexture(TextureUnit tu, intptr_t iTexture)
 {
 	//	g_DeviceCaps.MaxSimultaneousTextures = 1;
 	if (tu >= (int)g_DeviceCaps.MaxSimultaneousTextures) // not supported
@@ -1553,7 +1553,7 @@ RageDisplay_D3D::SetCullMode(CullMode mode)
 }
 
 void
-RageDisplay_D3D::DeleteTexture(unsigned iTexHandle)
+RageDisplay_D3D::DeleteTexture(intptr_t iTexHandle)
 {
 	if (iTexHandle == 0)
 		return;
@@ -1579,7 +1579,7 @@ RageDisplay_D3D::DeleteTexture(unsigned iTexHandle)
 		  g_TexResourceToTexturePalette.find(iTexHandle));
 }
 
-unsigned
+intptr_t
 RageDisplay_D3D::CreateTexture(RagePixelFormat pixfmt,
 							   RageSurface* img,
 							   bool bGenerateMipMaps)
@@ -1602,7 +1602,7 @@ RageDisplay_D3D::CreateTexture(RagePixelFormat pixfmt,
 							 RagePixelFormatToString(pixfmt).c_str(),
 							 GetErrorString(hr).c_str());
 
-	unsigned uTexHandle = (unsigned)pTex;
+	intptr_t uTexHandle = (intptr_t)pTex;
 
 	if (pixfmt == RagePixelFormat_PAL) {
 		// Save palette
@@ -1627,7 +1627,7 @@ RageDisplay_D3D::CreateTexture(RagePixelFormat pixfmt,
 }
 
 void
-RageDisplay_D3D::UpdateTexture(unsigned uTexHandle,
+RageDisplay_D3D::UpdateTexture(intptr_t uTexHandle,
 							   RageSurface* img,
 							   int xoffset,
 							   int yoffset,
@@ -1706,7 +1706,7 @@ class D3DRenderTarget_FramebufferObject : public RenderTarget
 	void Create(const RenderTargetParam& param,
 				int& iTextureWidthOut,
 				int& iTextureHeightOut) override;
-	unsigned GetTexture() const override { return (unsigned)m_uTexHandle; }
+	intptr_t GetTexture() const override { return (intptr_t)m_uTexHandle; }
 	void StartRenderingTo() override;
 	void FinishRenderingTo() override;
 
@@ -1812,7 +1812,7 @@ D3DRenderTarget_FramebufferObject::FinishRenderingTo()
 		LOG->Warn("Failed to set targetDepth to BackBufferDepth");
 }
 
-unsigned
+intptr_t
 RageDisplay_D3D::CreateRenderTarget(const RenderTargetParam& param,
 									int& iTextureWidthOut,
 									int& iTextureHeightOut)
@@ -1821,18 +1821,18 @@ RageDisplay_D3D::CreateRenderTarget(const RenderTargetParam& param,
 
 	pTarget->Create(param, iTextureWidthOut, iTextureHeightOut);
 
-	unsigned uTexture = pTarget->GetTexture();
+	intptr_t uTexture = pTarget->GetTexture();
 
 	ASSERT(g_mapRenderTargets.find(uTexture) == g_mapRenderTargets.end());
 	g_mapRenderTargets[uTexture] = pTarget;
 
 	return uTexture;
 }
 
-unsigned
+intptr_t
 RageDisplay_D3D::GetRenderTarget()
 {
-	for (map<unsigned, RenderTarget*>::const_iterator it =
+	for (map<intptr_t, RenderTarget*>::const_iterator it =
 		   g_mapRenderTargets.begin();
 		 it != g_mapRenderTargets.end();
 		 ++it)
@@ -1842,7 +1842,7 @@ RageDisplay_D3D::GetRenderTarget()
 }
 
 void
-RageDisplay_D3D::SetRenderTarget(unsigned uTexHandle, bool bPreserveTexture)
+RageDisplay_D3D::SetRenderTarget(intptr_t uTexHandle, bool bPreserveTexture)
 {
 	if (uTexHandle == 0) {
 		g_pd3dDevice->SetRenderState(D3DRS_CULLMODE, D3DCULL_CCW);

src/RageDisplay_D3D.h

@@ -25,19 +25,19 @@ class RageDisplay_D3D : public RageDisplay
 							   bool realtime = false) override;
 	bool SupportsThreadedRendering() override;
 	bool SupportsPerVertexMatrixScale() override { return false; }
-	unsigned CreateTexture(RagePixelFormat pixfmt,
+	intptr_t CreateTexture(RagePixelFormat pixfmt,
 						   RageSurface* img,
 						   bool bGenerateMipMaps) override;
-	void UpdateTexture(unsigned iTexHandle,
+	void UpdateTexture(intptr_t iTexHandle,
 					   RageSurface* img,
 					   int xoffset,
 					   int yoffset,
 					   int width,
 					   int height) override;
-	void DeleteTexture(unsigned iTexHandle) override;
+	void DeleteTexture(intptr_t iTexHandle) override;
 	void ClearAllTextures() override;
 	int GetNumTextureUnits() override;
-	void SetTexture(TextureUnit tu, unsigned iTexture) override;
+	void SetTexture(TextureUnit tu, intptr_t iTexture) override;
 	void SetTextureMode(TextureUnit tu, TextureMode tm) override;
 	void SetTextureWrapping(TextureUnit tu, bool b) override;
 	int GetMaxTextureSize() const override;
@@ -63,11 +63,11 @@ class RageDisplay_D3D : public RageDisplay
 							 const RageColor& specular,
 							 const RageVector3& dir) override;
 
-	unsigned CreateRenderTarget(const RenderTargetParam& param,
+	intptr_t CreateRenderTarget(const RenderTargetParam& param,
 								int& iTextureWidthOut,
 								int& iTextureHeightOut) override;
-	unsigned GetRenderTarget() override;
-	void SetRenderTarget(unsigned uTexHandle, bool bPreserveTexture) override;
+	intptr_t GetRenderTarget() override;
+	void SetRenderTarget(intptr_t uTexHandle, bool bPreserveTexture) override;
 
 	void SetSphereEnvironmentMapping(TextureUnit tu, bool b) override;
 	void SetCelShaded(int stage) override;
@@ -113,7 +113,7 @@ class RenderTarget
 						int& iTextureWidthOut,
 						int& iTextureHeightOut) = 0;
 
-	virtual unsigned GetTexture() const = 0;
+	virtual intptr_t GetTexture() const = 0;
 
 	/* Render to this RenderTarget. */
 	virtual void StartRenderingTo() = 0;

src/RageDisplay_Null.h

@@ -28,13 +28,13 @@ class RageDisplay_Null : public RageDisplay
 		return true;
 	}
 	bool SupportsPerVertexMatrixScale() override { return false; }
-	unsigned CreateTexture(RagePixelFormat,
+	intptr_t CreateTexture(RagePixelFormat,
 						   RageSurface* /* img */,
 						   bool /* bGenerateMipMaps */) override
 	{
 		return 1;
 	}
-	void UpdateTexture(unsigned /* iTexHandle */,
+	void UpdateTexture(intptr_t /* iTexHandle */,
 					   RageSurface* /* img */,
 					   int /* xoffset */,
 					   int /* yoffset */,
@@ -43,10 +43,10 @@ class RageDisplay_Null : public RageDisplay
 					   ) override
 	{
 	}
-	void DeleteTexture(unsigned /* iTexHandle */) override {}
+	void DeleteTexture(intptr_t /* iTexHandle */) override {}
 	void ClearAllTextures() override {}
 	int GetNumTextureUnits() override { return 1; }
-	void SetTexture(TextureUnit, unsigned /* iTexture */) override {}
+	void SetTexture(TextureUnit, intptr_t /* iTexture */) override {}
 	void SetTextureMode(TextureUnit, TextureMode) override {}
 	void SetTextureWrapping(TextureUnit, bool) override {}
 	int GetMaxTextureSize() const override { return 2048; }