Clean up POSIX streaming code

src/core/CdStream.h

@@ -43,6 +43,6 @@ char *CdStreamGetImageName(int32 cd);
 void CdStreamRemoveImages(void);
 int32 CdStreamGetNumImages(void);
 
-#ifndef _WIN32
+#ifdef FLUSHABLE_STREAMING
 extern bool flushStream[MAX_CDCHANNELS];
 #endif

src/core/CdStreamPosix.cpp

@@ -21,9 +21,9 @@
 #define CDDEBUG(f, ...)   debug ("%s: " f "\n", "cdvd_stream", ## __VA_ARGS__)
 #define CDTRACE(f, ...)   printf("%s: " f "\n", "cdvd_stream", ## __VA_ARGS__)
 
-// #define ONE_THREAD_PER_CHANNEL // Don't use if you're not on SSD/Flash. (Also you may want to benefit from this via using all channels in Streaming.cpp)
+#ifdef FLUSHABLE_STREAMING
-
 bool flushStream[MAX_CDCHANNELS];
+#endif
 
 struct CdReadInfo
 {
@@ -99,6 +99,7 @@ CdStreamInitThread(void)
 				ASSERT(0);
 				return;
 			}
+
 #ifdef ONE_THREAD_PER_CHANNEL
 			sprintf(semName,"/semaphore_start%d",i);
 			gpReadInfo[i].pStartSemaphore = sem_open(semName, O_CREAT, 0644, 1);
@@ -245,10 +246,12 @@ CdStreamRead(int32 channel, void *buffer, uint32 offset, uint32 size)
 	if ( pChannel->nSectorsToRead != 0 || pChannel->bReading ) {
 		if (pChannel->hFile == hImage - 1 && pChannel->nSectorOffset == _GET_OFFSET(offset) && pChannel->nSectorsToRead >= size)
 			return STREAM_SUCCESS;
-			
+#ifdef FLUSHABLE_STREAMING
 		flushStream[channel] = 1;
 		CdStreamSync(channel);
-		//return STREAM_NONE;
+#else
+		return STREAM_NONE;
+#endif
 	}
 
 	pChannel->hFile = hImage - 1;
@@ -316,34 +319,34 @@ CdStreamSync(int32 channel)
 	CdReadInfo *pChannel = &gpReadInfo[channel];
 	ASSERT( pChannel != nil );
 
+#ifdef FLUSHABLE_STREAMING
 	if (flushStream[channel]) {
-#ifdef ONE_THREAD_PER_CHANNEL
 		pChannel->nSectorsToRead = 0;
+#ifdef ONE_THREAD_PER_CHANNEL
 		pthread_kill(pChannel->pChannelThread, SIGUSR1);
 		if (pChannel->bReading) {
 			pChannel->bLocked = true;
-			while (pChannel->bLocked)
-				sem_wait(pChannel->pDoneSemaphore);
-		}
 #else
-		pChannel->nSectorsToRead = 0;
 		if (pChannel->bReading) {
 			pChannel->bLocked = true;
 			pthread_kill(_gCdStreamThread, SIGUSR1);
+#endif
 			while (pChannel->bLocked)
 				sem_wait(pChannel->pDoneSemaphore);
 		}
-#endif
 		pChannel->bReading = false;
 		flushStream[channel] = false;
 		return STREAM_NONE;
 	}
+#endif
 
 	if ( pChannel->nSectorsToRead != 0 )
 	{
 		pChannel->bLocked = true;
-		while (pChannel->bLocked)
+		while (pChannel->bLocked && pChannel->nSectorsToRead != 0){
 			sem_wait(pChannel->pDoneSemaphore);
+		}
+		pChannel->bLocked = false;
 	}
 
 	pChannel->bReading = false;
@@ -447,7 +450,7 @@ void *CdStreamThread(void *param)
 		if ( pChannel->bLocked )
 		{
 			pChannel->bLocked = 0;
-			sem_post(pChannel->pDoneSemaphore);	
+			sem_post(pChannel->pDoneSemaphore);
 		}
 		pChannel->bReading = false;
 	}
@@ -524,7 +527,9 @@ void
 CdStreamRemoveImages(void)
 {
 	for ( int32 i = 0; i < gNumChannels; i++ ) {
+#ifdef FLUSHABLE_STREAMING
 		flushStream[i] = 1;
+#endif
 		CdStreamSync(i);
 	}
 

src/core/Streaming.cpp

@@ -207,11 +207,15 @@ CStreaming::Init2(void)
 
 	// allocate streaming buffers
 	if(ms_streamingBufferSize & 1) ms_streamingBufferSize++;
+#ifndef ONE_THREAD_PER_CHANNEL
 	ms_pStreamingBuffer[0] = (int8*)RwMallocAlign(ms_streamingBufferSize*CDSTREAM_SECTOR_SIZE, CDSTREAM_SECTOR_SIZE);
 	ms_streamingBufferSize /= 2;
 	ms_pStreamingBuffer[1] = ms_pStreamingBuffer[0] + ms_streamingBufferSize*CDSTREAM_SECTOR_SIZE;
-#ifdef ONE_THREAD_PER_CHANNEL
+#else
-	ms_pStreamingBuffer[2] = (int8*)RwMallocAlign(ms_streamingBufferSize*2*CDSTREAM_SECTOR_SIZE, CDSTREAM_SECTOR_SIZE);
+	ms_pStreamingBuffer[0] = (int8*)RwMallocAlign(ms_streamingBufferSize*2*CDSTREAM_SECTOR_SIZE, CDSTREAM_SECTOR_SIZE);
+	ms_streamingBufferSize /= 2;
+	ms_pStreamingBuffer[1] = ms_pStreamingBuffer[0] + ms_streamingBufferSize*CDSTREAM_SECTOR_SIZE;
+	ms_pStreamingBuffer[2] = ms_pStreamingBuffer[1] + ms_streamingBufferSize*CDSTREAM_SECTOR_SIZE;
 	ms_pStreamingBuffer[3] = ms_pStreamingBuffer[2] + ms_streamingBufferSize*CDSTREAM_SECTOR_SIZE;
 #endif
 	debug("Streaming buffer size is %d sectors", ms_streamingBufferSize);
@@ -2305,9 +2309,10 @@ CStreaming::LoadRequestedModels(void)
 }
 
 
-// Let's load models first, then process it. Unfortunately processing models are still single-threaded.
+// Let's load models in 4 threads; when one of them becomes idle, process the file, and fill thread with another file. Unfortunately processing models are still single-threaded.
 // Currently only supported on POSIX streamer.
-#ifdef ONE_THREAD_PER_CHANNEL
+// WIP - some files are loaded swapped (CdStreamPosix problem?)
+#if 0 //def ONE_THREAD_PER_CHANNEL
 void
 CStreaming::LoadAllRequestedModels(bool priority)
 {
@@ -2326,14 +2331,18 @@ CStreaming::LoadAllRequestedModels(bool priority)
 	int streamIds[ARRAY_SIZE(ms_pStreamingBuffer)];
 	int streamSizes[ARRAY_SIZE(ms_pStreamingBuffer)];
 	int streamPoses[ARRAY_SIZE(ms_pStreamingBuffer)];
-	bool first = true;
+	int readOrder[4] = {-1}; // Channel IDs ordered by read time
+	int readI = 0;
 	int processI = 0;
+	bool first = true;
+
+	// All those "first" checks are because of variables aren't initialized in first pass.
 
 	while (true) {
-		// Enumerate files and start reading
 		for (int i=0; i<ARRAY_SIZE(ms_pStreamingBuffer); i++) {
+
+			// Channel has file to load
 			if (!first && streamIds[i] != -1) {
-				processI = i;
 				continue;
 			}
 
@@ -2346,12 +2355,16 @@ CStreaming::LoadAllRequestedModels(bool priority)
 
 				if (ms_aInfoForModel[streamId].GetCdPosnAndSize(posn, size)) {
 					streamIds[i] = -1;
+
+					// Big file, needs 2 buffer
 					if (size > (uint32)ms_streamingBufferSize) {
 						if (i + 1 == ARRAY_SIZE(ms_pStreamingBuffer))
-							continue;
+							break;
 						else if (!first && streamIds[i+1] != -1)
 							continue;
+
 					} else {
+						// Buffer of current channel is part of a "big file", pass
 						if (i != 0 && streamIds[i-1] != -1 && streamSizes[i-1] > (uint32)ms_streamingBufferSize)
 							continue;
 					}
@@ -2361,8 +2374,18 @@ CStreaming::LoadAllRequestedModels(bool priority)
 					streamIds[i] = streamId;
 					streamSizes[i] = size;
 					streamPoses[i] = posn;
+
+					if (!first)
+						assert(readOrder[readI] == -1);
+
+					//printf("read: order %d, ch %d, id %d, size %d\n", readI, i, streamId, size);
+
 					CdStreamRead(i, ms_pStreamingBuffer[i], imgOffset+posn, size);
-					processI = i;
+					readOrder[readI] = i;
+					if (first && readI+1 != ARRAY_SIZE(readOrder))
+						readOrder[readI+1] = -1;
+
+					readI = (readI + 1) % ARRAY_SIZE(readOrder);
 				} else {
 					ms_aInfoForModel[streamId].RemoveFromList();
 					DecrementRef(streamId);
@@ -2370,33 +2393,40 @@ CStreaming::LoadAllRequestedModels(bool priority)
 					ms_aInfoForModel[streamId].m_loadState = STREAMSTATE_LOADED;
 					streamIds[i] = -1;
 				}
-			} else
+			} else {
 				streamIds[i] = -1;
+				break;
+			}
 		}
 
 		first = false;
+		int nextChannel = readOrder[processI];
 
-		// Now process
+		// Now start processing
-		if (streamIds[processI] == -1) 
+		if (nextChannel == -1 || streamIds[nextChannel] == -1)
 			break;
 
-		// Try again on error
+		//printf("process: order %d, ch %d, id %d\n", processI, nextChannel, streamIds[nextChannel]);
-		while (CdStreamSync(processI) != STREAM_NONE) {
-			CdStreamRead(processI, ms_pStreamingBuffer[processI], imgOffset+streamPoses[processI], streamSizes[processI]);
-		}
-		ms_aInfoForModel[streamIds[processI]].m_loadState = STREAMSTATE_READING;
-		
-		MakeSpaceFor(streamSizes[processI] * CDSTREAM_SECTOR_SIZE);
-		ConvertBufferToObject(ms_pStreamingBuffer[processI], streamIds[processI]);
-		if(ms_aInfoForModel[streamIds[processI]].m_loadState == STREAMSTATE_STARTED)
-			FinishLoadingLargeFile(ms_pStreamingBuffer[processI], streamIds[processI]);
 
-		if(streamIds[processI] < STREAM_OFFSET_TXD){
+		// Try again on error
-			CSimpleModelInfo *mi = (CSimpleModelInfo*)CModelInfo::GetModelInfo(streamIds[processI]);
+		while (CdStreamSync(nextChannel) != STREAM_NONE) {
+			CdStreamRead(nextChannel, ms_pStreamingBuffer[nextChannel], imgOffset+streamPoses[nextChannel], streamSizes[nextChannel]);
+		}
+		ms_aInfoForModel[streamIds[nextChannel]].m_loadState = STREAMSTATE_READING;
+
+		MakeSpaceFor(streamSizes[nextChannel] * CDSTREAM_SECTOR_SIZE);
+		ConvertBufferToObject(ms_pStreamingBuffer[nextChannel], streamIds[nextChannel]);
+		if(ms_aInfoForModel[streamIds[nextChannel]].m_loadState == STREAMSTATE_STARTED)
+			FinishLoadingLargeFile(ms_pStreamingBuffer[nextChannel], streamIds[nextChannel]);
+
+		if(streamIds[nextChannel] < STREAM_OFFSET_TXD){
+			CSimpleModelInfo *mi = (CSimpleModelInfo*)CModelInfo::GetModelInfo(streamIds[nextChannel]);
 			if(mi->IsSimple())
 				mi->m_alpha = 255;
 		}
-		streamIds[processI] = -1;
+		streamIds[nextChannel] = -1;
+		readOrder[processI] = -1;
+		processI = (processI + 1) % ARRAY_SIZE(readOrder);
 	}
 
 	ms_bLoadingBigModel = false;
@@ -2443,7 +2473,7 @@ CStreaming::LoadAllRequestedModels(bool priority)
 				status = CdStreamRead(0, ms_pStreamingBuffer[0], imgOffset+posn, size);
 			while(CdStreamSync(0) || status == STREAM_NONE);
 			ms_aInfoForModel[streamId].m_loadState = STREAMSTATE_READING;
-			
+
 			MakeSpaceFor(size * CDSTREAM_SECTOR_SIZE);
 			ConvertBufferToObject(ms_pStreamingBuffer[0], streamId);
 			if(ms_aInfoForModel[streamId].m_loadState == STREAMSTATE_STARTED)
@@ -2500,7 +2530,7 @@ CStreaming::FlushRequestList(void)
 		next = si->m_next;
 		RemoveModel(si - ms_aInfoForModel);
 	}
-#ifndef _WIN32
+#ifdef FLUSHABLE_STREAMING
 	if(ms_channel[0].state == CHANNELSTATE_READING) {
 		flushStream[0] = 1;
 	}
@@ -3216,4 +3246,4 @@ CStreaming::PrintStreamingBufferState()
 		DoRWStuffEndOfFrame();
 	}
 	CTimer::Update();
-}
+}

src/core/Streaming.h

@@ -88,7 +88,11 @@ public:
 	static int32 ms_oldSectorX;
 	static int32 ms_oldSectorY;
 	static int32 ms_streamingBufferSize;
+#ifndef ONE_THREAD_PER_CHANNEL
 	static int8 *ms_pStreamingBuffer[2];
+#else
+	static int8 *ms_pStreamingBuffer[4];
+#endif
 	static size_t ms_memoryUsed;
 	static CStreamingChannel ms_channel[2];
 	static int32 ms_channelError;

src/core/config.h

@@ -393,11 +393,12 @@ static_assert(false, "SUPPORT_XBOX_SCRIPT and SUPPORT_MOBILE_SCRIPT are mutually
 
 #endif
 
-#ifdef LIBRW
+// Streaming
-// these are not supported with librw yet
+#if !defined(_WIN32) && !defined(__SWITCH__)
+	//#define ONE_THREAD_PER_CHANNEL // Don't use if you're not on SSD/Flash - also not utilized too much right now(see commented LoadAllRequestedModels in Streaming.cpp)
+	#define FLUSHABLE_STREAMING // Make it possible to interrupt reading when processing file isn't needed anymore.
 #endif
-// IMG
+#define BIG_IMG // Not complete - allows to read larger img files
-#define BIG_IMG // allows to read larger img files
 
 //#define SQUEEZE_PERFORMANCE
 #ifdef SQUEEZE_PERFORMANCE
@@ -405,6 +406,8 @@ static_assert(false, "SUPPORT_XBOX_SCRIPT and SUPPORT_MOBILE_SCRIPT are mutually
 	#undef NO_ISLAND_LOADING
 #endif
 
+// -------
+
 #if defined __MWERKS__ || defined VANILLA_DEFINES
 #define FINAL
 #undef CHATTYSPLASH

src/skel/glfw/glfw.cpp

@@ -1271,10 +1271,11 @@ void terminateHandler(int sig, siginfo_t *info, void *ucontext) {
 	RsGlobal.quit = TRUE;
 }
 
+#ifdef FLUSHABLE_STREAMING
 void dummyHandler(int sig){
 	// Don't kill the app pls
 }
-
+#endif
 #endif
 
 void resizeCB(GLFWwindow* window, int width, int height) {
@@ -1528,11 +1529,13 @@ main(int argc, char *argv[])
 	act.sa_sigaction = terminateHandler;
 	act.sa_flags = SA_SIGINFO;
 	sigaction(SIGTERM, &act, NULL);
+#ifdef FLUSHABLE_STREAMING
 	struct sigaction sa;
 	sigemptyset(&sa.sa_mask);
 	sa.sa_handler = dummyHandler;
 	sa.sa_flags = 0;
-	sigaction(SIGUSR1, &sa, NULL); // Needed for CdStreamPosix
+	sigaction(SIGUSR1, &sa, NULL);
+#endif
 #endif
 
 	/*