Skip to content

Commit d0d8e54

Browse files
committed
2026.2.14.0
TokenBatch: add 'MyWorkingDirectory' property API.Instagram: update 'ID' extraction; reduce the number of downloaded stories (GQL) to 5 API.Twitter: get a new username based on the user ID API.XHamster: videos aren't downloading Feed: fix a bug when removing from favorites
1 parent 164b999 commit d0d8e54

12 files changed

Lines changed: 287 additions & 114 deletions

File tree

Changelog.md

Lines changed: 19 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2,13 +2,30 @@
22
- [ffmpeg](https://github.com/AAndyProgram/SCrawler/wiki/Settings#ffmpeg)
33
- x64 version - [release](https://github.com/GyanD/codexffmpeg/releases/tag/5.1.2); [zip](https://github.com/GyanD/codexffmpeg/releases/download/5.1.2/ffmpeg-5.1.2-full_build.zip); **version `5.1.2-full_build-www.gyan.dev`**
44
- x86 version - [release](https://github.com/yt-dlp/FFmpeg-Builds/releases/tag/autobuild-2022-11-30-12-57); [zip](https://github.com/yt-dlp/FFmpeg-Builds/releases/download/autobuild-2022-11-30-12-57/ffmpeg-N-109274-gd7a5f068c2-win32-gpl.zip); **version `N-109457-geeb280f351-20221226`**
5-
- [Gallery-dl](https://github.com/AAndyProgram/SCrawler/wiki/Settings#gallery-dl) - **1.31.4**
6-
- [YT-DLP](https://github.com/AAndyProgram/SCrawler/wiki/Settings#yt-dlp) - **2025.12.08**
5+
- [Gallery-dl](https://github.com/AAndyProgram/SCrawler/wiki/Settings#gallery-dl) - **1.31.6**
6+
- [YT-DLP](https://github.com/AAndyProgram/SCrawler/wiki/Settings#yt-dlp) - **2026.02.04.233607**
77
- [Deno](https://github.com/AAndyProgram/SCrawler/wiki/Settings#deno) - latest *(`2.0.0` or higher)*
88
- [OF-Scraper](https://github.com/AAndyProgram/SCrawler/wiki/Settings#of-scraper) - **3.12.9** ([release](https://github.com/datawhores/OF-Scraper/releases/tag/3.12.9))
99

1010
# 2026
1111

12+
## 2026.2.14.0
13+
14+
*2026-02-14*
15+
16+
- Added
17+
- Sites:
18+
- Twitter: get a new username based on the user ID
19+
- Minor improvements
20+
- Updated
21+
- gallery-dl up to version **1.31.6**
22+
- yt-dlp up to version **2026.02.04.233607**
23+
- Fixed
24+
- Sites:
25+
- **Instagram: some profiles aren't downloading**
26+
- xHamster: videos aren't downloading
27+
- Minor bugs
28+
1229
## 2026.1.24.0
1330

1431
*2026-01-24*

SCrawler/API/Base/TokenBatch.vb

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,12 +13,17 @@ Namespace API.Base
1313
Friend Property TempPostsList As List(Of String)
1414
Protected ReadOnly Token As CancellationToken
1515
Friend Property DebugMode As Boolean = False
16+
Friend Overridable Property MyWorkingDirectory As SFile = Nothing
1617
Friend Sub New(ByVal _Token As CancellationToken, Optional ByVal _MainProcessName As String = Nothing, Optional ByVal WorkingDir As SFile = Nothing)
1718
MyBase.New(True)
1819
Token = _Token
1920
MainProcessName = _MainProcessName
2021
If Not WorkingDir.IsEmptyString Then ChangeDirectory(WorkingDir)
2122
End Sub
23+
Public Overrides Sub ChangeDirectory(ByVal Directory As SFile)
24+
MyBase.ChangeDirectory(Directory)
25+
If Not Directory.IsEmptyString Then MyWorkingDirectory = Directory
26+
End Sub
2227
Protected Overrides Function Internal_Execute(ByVal Commands As IEnumerable(Of String), ByVal e As ErrorsDescriber) As Boolean
2328
If Not Encoding.HasValue Then Encoding = UnicodeEncoding
2429
Return MyBase.Internal_Execute(Commands, e)

SCrawler/API/Instagram/Declarations.vb

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@ Namespace API.Instagram
2020
Friend Const PageTokenRegexPatternDefault As String = "\[\],{""token"":""(.*?)""},\d+\]"
2121
Friend ReadOnly Regex_UserToken_dtsg As RParams = RParams.DMS("DTSGInitialData["":,.\[\]]*?{\s*.token.:\s*""([^""]+)", 1, EDP.ReturnValue)
2222
Friend ReadOnly Regex_UserToken_lsd As RParams = RParams.DMS("LSD["":,.\[\]]*?{\s*.token.:\s*""([^""]+)", 1, EDP.ReturnValue)
23+
Friend ReadOnly Regex_ProfileID As RParams = RParams.DMS("profilePage_(\d+)", 1, EDP.ReturnValue)
2324
Friend Sub UpdateResponser(ByVal Source As IResponse, ByRef Destination As Responser, ByVal UpdateWwwClaim As Boolean)
2425
Const r_wwwClaimName$ = "x-ig-set-www-claim"
2526
Const r_tokenName$ = SiteSettings.Header_CSRF_TOKEN_COOKIE

SCrawler/API/Instagram/EditorExchangeOptions.vb

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,8 @@ Namespace API.Instagram
3636
<PSetting(Caption:="Place the extracted image into the video folder")>
3737
Friend Property PutImageVideoFolder As Boolean
3838
Friend Overrides Property UserName As String
39+
<PSetting(Address:=SettingAddress.User, Caption:="Verified profile", ToolTip:="This profile has a verified mark")>
40+
Friend Property IsVerifiedProfile As Boolean = False
3941
<PSetting(Address:=SettingAddress.User, Caption:="Force update UserName", ToolTip:="Try to force update UserName if it is not found on the site")>
4042
Friend Property ForceUpdateUserName As Boolean = False
4143
<PSetting(Address:=SettingAddress.User, Caption:="Force update user information")>
@@ -57,6 +59,8 @@ Namespace API.Instagram
5759

5860
PutImageVideoFolder = .PutImageVideoFolder
5961

62+
IsVerifiedProfile = .IsVerifiedProfile
63+
6064
ForceUpdateUserName = .ForceUpdateUserName
6165
ForceUpdateUserInfo = .ForceUpdateUserInfo
6266
End With

SCrawler/API/Instagram/SiteSettings.vb

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -148,6 +148,8 @@ Namespace API.Instagram
148148
#End Region
149149
<PropertyOption(ControlText:="Use GraphQL to download", IsAuth:=True), PXML, PClonable>
150150
Friend ReadOnly Property USE_GQL As PropertyValue
151+
<PropertyOption(ControlText:="Use GraphQL to download user data", IsAuth:=True), PXML, PClonable, HiddenControl>
152+
Friend ReadOnly Property USE_GQL_UserData As PropertyValue
151153
#End Region
152154
#Region "Download data"
153155
<PropertyOption(ControlText:="Download timeline", Category:=CAT_DOWN), PXML, PClonable>
@@ -165,6 +167,14 @@ Namespace API.Instagram
165167
<PropertyOption(ControlText:="Download tagged posts", Category:=CAT_DOWN), PXML, PClonable>
166168
Friend ReadOnly Property DownloadTagged As PropertyValue
167169
<PXML> Private ReadOnly Property DownloadTagged_Def As PropertyValue
170+
<PropertyOption(ControlText:="Number of posts (verified)", ControlToolTip:="The number of posts received per request if the profile has a verified mark", Category:=CAT_DOWN), PXML, PClonable, HiddenControl>
171+
Friend ReadOnly Property PostNumberVerified As PropertyValue
172+
<Provider(NameOf(PostNumberVerified), FieldsChecker:=True)>
173+
Private ReadOnly Property PostNumberVerifiedProvider As IFormatProvider
174+
<PropertyOption(ControlText:="Number of posts (unverified)", ControlToolTip:="The number of posts received per request if the profile doesn't have a verified mark", Category:=CAT_DOWN), PXML, PClonable, HiddenControl>
175+
Friend ReadOnly Property PostNumberVerifiedNot As PropertyValue
176+
<Provider(NameOf(PostNumberVerifiedNot), FieldsChecker:=True)>
177+
Private ReadOnly Property PostNumberVerifiedNotProvider As IFormatProvider
168178
#End Region
169179
#Region "Timers"
170180
Friend Const TimersUrgentTip As String = vbCr & "It is highly recommended not to change the default value."
@@ -485,6 +495,7 @@ Namespace API.Instagram
485495
HH_IG_WWW_CLAIM_USE_DEFAULT_ALGO = New PropertyValue(True)
486496
TokenUpdateIntervalProvider = New TokenRefreshIntervalProvider
487497
USE_GQL = New PropertyValue(False)
498+
USE_GQL_UserData = New PropertyValue(True)
488499

489500
DownloadTimeline = New PropertyValue(True)
490501
DownloadTimeline_Def = New PropertyValue(DownloadTimeline.Value, GetType(Boolean))
@@ -496,6 +507,10 @@ Namespace API.Instagram
496507
DownloadStoriesUser_Def = New PropertyValue(DownloadStoriesUser.Value, GetType(Boolean))
497508
DownloadTagged = New PropertyValue(False)
498509
DownloadTagged_Def = New PropertyValue(DownloadTagged.Value, GetType(Boolean))
510+
PostNumberVerified = New PropertyValue(50)
511+
PostNumberVerifiedProvider = New TimersChecker(12)
512+
PostNumberVerifiedNot = New PropertyValue(12)
513+
PostNumberVerifiedNotProvider = New TimersChecker(12)
499514

500515
RequestsWaitTimer_Any = New PropertyValue(1000)
501516
RequestsWaitTimer_AnyProvider = New TimersChecker(0)

SCrawler/API/Instagram/UserData.GQL.vb

Lines changed: 59 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -6,12 +6,13 @@
66
'
77
' This program is distributed in the hope that it will be useful,
88
' but WITHOUT ANY WARRANTY
9+
Imports System.Security.Cryptography
910
Imports System.Threading
10-
Imports SCrawler.API.Base
11-
Imports PersonalUtilities.Functions.XML
1211
Imports PersonalUtilities.Functions.RegularExpressions
12+
Imports PersonalUtilities.Functions.XML
1313
Imports PersonalUtilities.Tools.Web.Clients
1414
Imports PersonalUtilities.Tools.Web.Documents.JSON
15+
Imports SCrawler.API.Base
1516
Namespace API.Instagram
1617
Partial Friend Class UserData
1718
#Region "Tokens"
@@ -43,9 +44,9 @@ Namespace API.Instagram
4344
Private Const GQL_UserStories_DocId As String = "25231722019806941"
4445
Private Const GQL_UserStories_FbFriendlyName As String = "PolarisStoriesV3ReelPageStandaloneQuery"
4546

46-
Private Const GQL_Timeline_DocId As String = "7268577773270422"
47+
Private Const GQL_Timeline_DocId As String = "7268577773270422" '"34579740524958711" '"7268577773270422"
4748
Private Const GQL_Timeline_FbFriendlyName As String = "PolarisProfilePostsQuery"
48-
Private Const GQL_Timeline_DocId_Second As String = "7286316061475375"
49+
Private Const GQL_Timeline_DocId_Second As String = "7286316061475375" '"33944389991841132" '"7286316061475375"
4950
Private Const GQL_Timeline_FbFriendlyName_Second As String = "PolarisProfilePostsTabContentQuery_connection"
5051

5152
Private Const GQL_Reels_DocId As String = "7191572580905225"
@@ -64,48 +65,61 @@ Namespace API.Instagram
6465
Responser.Headers.Add(GQL_HEADER_FB_FRINDLY_NAME, HeaderValue)
6566
Responser.Headers.Add(GQL_HEADER_FB_LSD, Token_lsd)
6667
End Sub
67-
<Obsolete("Use 'GET' function: 'GetUserData'", False)>
68-
Private Sub GetUserDataGQL(ByVal Token As CancellationToken)
68+
'<Obsolete("Use 'GET' function: 'GetUserData'", False)>
69+
Private Function GetUserDataGQL(ByVal Token As CancellationToken) As String
6970
Dim vars$ = String.Format(GQL_URL_PATTERN_VARS, GQL_UserData_DocId, Token_lsd, Token_dtsg_Var, GQL_UserData_FbFriendlyName,
7071
SymbolsConverter.ASCII.EncodeSymbolsOnly("{" & $"""id"":""{ID}"",""relay_header"":false,""render_surface"":""PROFILE""" & "}"))
7172
UpdateRequestNumber()
7273
ChangeResponserMode(True)
7374
UpdateHeadersGQL(GQL_UserData_FbFriendlyName)
7475
Dim r$ = Responser.GetResponse(GQL_URL, vars)
75-
If Not r.IsEmptyString Then
76-
Using j As EContainer = JsonDocument.Parse(r)
77-
If j.ListExists Then
78-
With j({"data", "user"})
79-
If .ListExists Then
80-
UserSiteName = .Value("full_name").IfNullOrEmpty(UserSiteName)
81-
Dim f As New SFile With {.Path = DownloadContentDefault_GetRootDir(), .Name = "ProfilePicture", .Extension = "jpg"}
82-
Dim pic$ = .Value({"hd_profile_pic_url_info"}, "url").IfNullOrEmpty(.Value("profile_pic_url"))
83-
If Not pic.IsEmptyString Then GetWebFile(pic, f, EDP.ReturnValue)
84-
UserDescriptionUpdate(.Value("biography"))
85-
End If
86-
End With
87-
End If
88-
End Using
89-
End If
90-
End Sub
76+
Return r
77+
'If Not r.IsEmptyString Then
78+
' Using j As EContainer = JsonDocument.Parse(r)
79+
' If j.ListExists Then
80+
' With j({"data", "user"})
81+
' If .ListExists Then
82+
' UserSiteName = .Value("full_name").IfNullOrEmpty(UserSiteName)
83+
' IsVerifiedProfile = .Value("is_verified").FromXML(Of Boolean)(False)
84+
' IsVerifiedProfile_Checked = True
85+
' Dim descr$ = .Value("biography")
86+
' If If(.Item("bio_links")?.Count, 0) > 0 Then descr.StringAppend(.Item("bio_links").Select(Function(bl) bl.Value("url")).ListToString(vbNewLine), vbNewLine)
87+
' Dim eUrl$ = .Value("external_url")
88+
' If Not eUrl.IsEmptyString AndAlso (descr.IsEmptyString OrElse Not descr.Contains(eUrl)) Then descr.StringAppendLine(eUrl)
89+
' UserDescriptionUpdate(descr)
90+
91+
' Dim f As New SFile With {.Path = DownloadContentDefault_GetRootDir(), .Name = "ProfilePicture", .Extension = "jpg"}
92+
' Dim pic$ = .Value({"hd_profile_pic_url_info"}, "url").IfNullOrEmpty(.Value("profile_pic_url"))
93+
' If Not pic.IsEmptyString Then GetWebFile(pic, f, EDP.ReturnValue)
94+
' End If
95+
' End With
96+
' End If
97+
' End Using
98+
'End If
99+
End Function
91100
Private Function GetTimelineGQL(ByVal Cursor As String, ByVal Token As CancellationToken) As String
92101
Const none_cursor$ = "none"
93-
Dim nextCursor$ = String.Empty, hasNextPage$ = String.Empty
102+
Dim nextCursor$ = String.Empty
103+
Dim hasNextPage As Boolean = False
94104
Dim vars$
95105

96106
ThrowAny(Token)
97107
UpdateRequestNumber()
98108
ChangeResponserMode(True)
99109

100110
If Cursor.IsEmptyString Then
101-
vars = "{""data"":{""count"":50,""include_relationship_info"":true,""latest_besties_reel_media"":true,""latest_reel_media"":true},""username"":""" &
111+
vars = "{""data"":{""count"":" & PostNumberPerRequest & ",""include_relationship_info"":true,""latest_besties_reel_media"":true,""latest_reel_media"":true},""username"":""" &
102112
NameTrue & """,""__relay_internal__pv__PolarisShareMenurelayprovider"":false}"
113+
'vars = "{""data"":{""count"":" & PostNumberPerRequest & ",""include_reel_media_seen_timestamp"":true,""include_relationship_info"":true,""latest_besties_reel_media"":true,""latest_reel_media"":true},""username"":""" &
114+
' NameTrue & """,""__relay_internal__pv__PolarisShareMenurelayprovider"":false}"
103115
vars = String.Format(GQL_URL_PATTERN_VARS, GQL_Timeline_DocId, Token_lsd, Token_dtsg_Var, GQL_Timeline_FbFriendlyName,
104116
SymbolsConverter.ASCII.EncodeSymbolsOnly(vars))
105117
UpdateHeadersGQL(GQL_Timeline_FbFriendlyName)
106118
Else
107-
vars = "{""after"":""" & Cursor & """,""before"":null,""data"":{""count"":50,""include_relationship_info"":true,""latest_besties_reel_media"":true,""latest_reel_media"":true},""first"":50,""last"":null,""username"":""" &
119+
vars = "{""after"":""" & Cursor & """,""before"":null,""data"":{""count"":" & PostNumberPerRequest & ",""include_relationship_info"":true,""latest_besties_reel_media"":true,""latest_reel_media"":true},""first"":" & PostNumberPerRequest & ",""last"":null,""username"":""" &
108120
NameTrue & """,""__relay_internal__pv__PolarisShareMenurelayprovider"":false}"
121+
'vars = "{""after"":""" & Cursor & """,""before"":null,""data"":{""count"":" & PostNumberPerRequest & ",""include_reel_media_seen_timestamp"":true,""include_relationship_info"":true,""latest_besties_reel_media"":true,""latest_reel_media"":true},""first"":" & PostNumberPerRequest & ",""last"":null,""username"":""" &
122+
' NameTrue & """}"
109123
vars = String.Format(GQL_URL_PATTERN_VARS, GQL_Timeline_DocId_Second, Token_lsd, Token_dtsg_Var, GQL_Timeline_FbFriendlyName_Second,
110124
SymbolsConverter.ASCII.EncodeSymbolsOnly(vars))
111125
UpdateHeadersGQL(GQL_Timeline_FbFriendlyName_Second)
@@ -140,7 +154,8 @@ Namespace API.Instagram
140154
End Function
141155
Private Function GetHighlightsGQL_List() As List(Of String)
142156

143-
Dim nextCursor$ = String.Empty, hasNextPage$ = String.Empty
157+
Dim nextCursor$ = String.Empty
158+
Dim hasNextPage As Boolean = False
144159
Dim i% = -1
145160
Dim hList As New List(Of String)
146161
Dim tmpList As New List(Of String)
@@ -178,7 +193,9 @@ Namespace API.Instagram
178193
Dim tmpList As New List(Of String)
179194
Dim i% = -1
180195
If StoriesList.ListExists Then
181-
tmpList.AddRange(StoriesList.Take(10))
196+
'TODO: 5 Instagram stories
197+
'tmpList.AddRange(StoriesList.Take(10))
198+
tmpList.AddRange(StoriesList.Take(5))
182199
StoriesList.RemoveRange(0, tmpList.Count)
183200

184201
Dim vars$ = String.Format(GQL_URL_PATTERN_VARS, GQL_Highlights_DocId_Second, Token_lsd, Token_dtsg_Var, GQL_Highlights_FbFriendlyName_Second,
@@ -238,11 +255,9 @@ Namespace API.Instagram
238255
Private Function GetReelsGQL(ByVal Cursor As String) As String
239256
GetReelsGQL_SetEnvir = True
240257

241-
Dim errData$ = String.Empty
242-
If Cursor.IsEmptyString And Not ValidateBaseTokens() Then GetPageTokens()
243-
If Cursor.IsEmptyString And Not ValidateBaseTokens(errData) Then ValidateBaseTokens_Error(errData)
258+
UpdateTokens(Cursor.IsEmptyString)
244259

245-
Dim vars$ = """data"":{""include_feed_video"":true,""page_size"":50,""target_user_id"":""" & ID & """}"
260+
Dim vars$ = """data"":{""include_feed_video"":true,""page_size"":" & PostNumberPerRequest & ",""target_user_id"":""" & ID & """}"
246261
If Not Cursor.IsEmptyString Then vars = $"""after"":""{Cursor}"",""before"":null,{vars},""first"":4,""last"":null"
247262
vars = String.Format(GQL_URL_PATTERN_VARS, GQL_Reels_DocId, Token_lsd, Token_dtsg_Var, GQL_Reels_FbFriendlyName,
248263
SymbolsConverter.ASCII.EncodeSymbolsOnly("{" & vars & "}"))
@@ -258,10 +273,10 @@ Namespace API.Instagram
258273
Dim vars$
259274
If Cursor.IsEmptyString Then
260275
vars = String.Format(GQL_URL_PATTERN_VARS, GQL_Tagged_DocId, Token_lsd, Token_dtsg_Var, GQL_Tagged_FbFriendlyName,
261-
SymbolsConverter.ASCII.EncodeSymbolsOnly("{" & $"""count"":50,""user_id"":""{ID}""" & "}"))
276+
SymbolsConverter.ASCII.EncodeSymbolsOnly("{" & $"""count"":{PostNumberPerRequest},""user_id"":""{ID}""" & "}"))
262277
Else
263278
vars = String.Format(GQL_URL_PATTERN_VARS, GQL_Tagged_DocId, Token_lsd, Token_dtsg_Var, GQL_Tagged_FbFriendlyName,
264-
SymbolsConverter.ASCII.EncodeSymbolsOnly("{" & $"""after"":""{Cursor}"",""before"":null,""count"":50,""first"":50,""last"":null,""user_id"":""{ID}""" & "}"))
279+
SymbolsConverter.ASCII.EncodeSymbolsOnly("{" & $"""after"":""{Cursor}"",""before"":null,""count"":{PostNumberPerRequest},""first"":{PostNumberPerRequest},""last"":null,""user_id"":""{ID}""" & "}"))
265280
End If
266281
UpdateRequestNumber()
267282
ChangeResponserMode(True)
@@ -270,6 +285,13 @@ Namespace API.Instagram
270285
End Function
271286
#End Region
272287
#Region "ValidateBaseTokens"
288+
Private Sub UpdateTokens(ByVal process As Boolean)
289+
If process Then
290+
Dim TokensErrData$ = String.Empty
291+
If Not ValidateBaseTokens() Then GetPageTokens()
292+
If Not ValidateBaseTokens(TokensErrData) Then ValidateBaseTokens_Error(TokensErrData)
293+
End If
294+
End Sub
273295
Protected Overridable Overloads Function ValidateBaseTokens() As Boolean
274296
Return ValidateBaseTokens(Nothing)
275297
End Function
@@ -307,6 +329,10 @@ Namespace API.Instagram
307329
Try
308330
If Not r.IsEmptyString Then
309331
ResetBaseTokens()
332+
If ID.IsEmptyString Then
333+
Dim __id$ = RegexReplace(r, Regex_ProfileID)
334+
If CLng(AConvert(Of Long)(__id, 0, EDP.ReturnValue)) <> 0 Then ID = __id
335+
End If
310336
Select Case Attempt
311337
Case 0
312338
Dim rr As RParams = RParams.DM(PageTokenRegexPatternDefault, 0, RegexReturn.List, EDP.ReturnValue)

0 commit comments

Comments
 (0)