@paladox opened this Issue on December 10th 2020 Contributor

I'm not sure if this is because of better tracking under matomo 4 or a bug/issue. See below photo (you can tell when we deploy matomo 4)

Screenshot 2020-12-10 at 13 37 44

We did disable cookies being created with disableCookies in the js code (though this was done like earlier this year under matomo 3).

Our config is:

; <?php exit; ?> DO NOT REMOVE THIS LINE
; file automatically generated or modified by Matomo; you can manually override the default values in global.ini.php by redefining them in this file.
[database]
host = "db13.miraheze.org"
username = "piwik"
password = "<%= <a class='mention' href='https://github.com/password'>@password</a> %>"
dbname = "piwik"
tables_prefix = "piwik_"
charset = "utf8mb4"
enable_ssl = 1
ssl_ca = "/etc/ssl/certs/Sectigo.crt"
adapter = MYSQLI

[General]
force_ssl = 1
salt = "<%= <a class='mention' href='https://github.com/salt'>@salt</a> %>"
trusted_hosts[] = "matomo.miraheze.org"
enable_load_data_infile = 0
; disable browser trigger archiving for all requests (even those with a segment)
browser_archiving_disabled_enforce = 1

[mail]
transport = "smtp"
port = 25
host = "mail.miraheze.org"
type = "LOGIN"
username = "noreply"
password = "<%= <a class='mention' href='https://github.com/noreply_password'>@noreply_password</a> %>"
encryption = "tls"

[Plugins]
Plugins[] = "CorePluginsAdmin"
Plugins[] = "CoreAdminHome"
Plugins[] = "CoreHome"
Plugins[] = "WebsiteMeasurable"
Plugins[] = "Diagnostics"
Plugins[] = "CoreVisualizations"
Plugins[] = "Proxy"
Plugins[] = "API"
Plugins[] = "ExamplePlugin"
Plugins[] = "Widgetize"
Plugins[] = "Transitions"
Plugins[] = "LanguagesManager"
Plugins[] = "Actions"
Plugins[] = "Dashboard"
Plugins[] = "MultiSites"
Plugins[] = "Referrers"
Plugins[] = "UserLanguage"
Plugins[] = "DevicesDetection"
Plugins[] = "Goals"
Plugins[] = "Ecommerce"
Plugins[] = "SEO"
Plugins[] = "Events"
Plugins[] = "UserCountry"
Plugins[] = "GeoIp2"
Plugins[] = "VisitsSummary"
Plugins[] = "VisitFrequency"
Plugins[] = "VisitTime"
Plugins[] = "VisitorInterest"
Plugins[] = "ExampleAPI"
Plugins[] = "RssWidget"
Plugins[] = "Feedback"
Plugins[] = "Monolog"
Plugins[] = "TwoFactorAuth"
Plugins[] = "UsersManager"
Plugins[] = "SitesManager"
Plugins[] = "Installation"
Plugins[] = "CoreUpdater"
Plugins[] = "CoreConsole"
Plugins[] = "ScheduledReports"
Plugins[] = "UserCountryMap"
Plugins[] = "Live"
Plugins[] = "CustomVariables"
Plugins[] = "PrivacyManager"
Plugins[] = "ImageGraph"
Plugins[] = "Annotations"
Plugins[] = "MobileMessaging"
Plugins[] = "Overlay"
Plugins[] = "SegmentEditor"
Plugins[] = "Insights"
Plugins[] = "Morpheus"
Plugins[] = "Contents"
Plugins[] = "TestRunner"
Plugins[] = "BulkTracking"
Plugins[] = "Resolution"
Plugins[] = "DevicePlugins"
Plugins[] = "Heartbeat"
Plugins[] = "Intl"
Plugins[] = "Marketplace"
Plugins[] = "ProfessionalServices"
Plugins[] = "UserId"
Plugins[] = "QueuedTracking"
Plugins[] = "DeviceDetectorCache"
Plugins[] = "LoginLdap"
Plugins[] = "Login"

[PluginsInstalled]
PluginsInstalled[] = "Diagnostics"
PluginsInstalled[] = "CoreAdminHome"
PluginsInstalled[] = "UsersManager"
PluginsInstalled[] = "SitesManager"
PluginsInstalled[] = "Installation"
PluginsInstalled[] = "Monolog"
PluginsInstalled[] = "Intl"
PluginsInstalled[] = "CorePluginsAdmin"
PluginsInstalled[] = "CoreHome"
PluginsInstalled[] = "WebsiteMeasurable"
PluginsInstalled[] = "CoreVisualizations"
PluginsInstalled[] = "Proxy"
PluginsInstalled[] = "API"
PluginsInstalled[] = "ExamplePlugin"
PluginsInstalled[] = "Widgetize"
PluginsInstalled[] = "Transitions"
PluginsInstalled[] = "LanguagesManager"
PluginsInstalled[] = "Actions"
PluginsInstalled[] = "Dashboard"
PluginsInstalled[] = "MultiSites"
PluginsInstalled[] = "Referrers"
PluginsInstalled[] = "UserLanguage"
PluginsInstalled[] = "DevicesDetection"
PluginsInstalled[] = "Goals"
PluginsInstalled[] = "SEO"
PluginsInstalled[] = "Events"
PluginsInstalled[] = "UserCountry"
PluginsInstalled[] = "VisitsSummary"
PluginsInstalled[] = "VisitFrequency"
PluginsInstalled[] = "VisitTime"
PluginsInstalled[] = "VisitorInterest"
PluginsInstalled[] = "ExampleAPI"
PluginsInstalled[] = "ExampleRssWidget"
PluginsInstalled[] = "Provider"
PluginsInstalled[] = "CoreUpdater"
PluginsInstalled[] = "CoreConsole"
PluginsInstalled[] = "ScheduledReports"
PluginsInstalled[] = "UserCountryMap"
PluginsInstalled[] = "Live"
PluginsInstalled[] = "CustomVariables"
PluginsInstalled[] = "PrivacyManager"
PluginsInstalled[] = "ImageGraph"
PluginsInstalled[] = "Annotations"
PluginsInstalled[] = "MobileMessaging"
PluginsInstalled[] = "Overlay"
PluginsInstalled[] = "SegmentEditor"
PluginsInstalled[] = "Insights"
PluginsInstalled[] = "ZenMode"
PluginsInstalled[] = "LeftMenu"
PluginsInstalled[] = "Morpheus"
PluginsInstalled[] = "Contents"
PluginsInstalled[] = "TestRunner"
PluginsInstalled[] = "BulkTracking"
PluginsInstalled[] = "Resolution"
PluginsInstalled[] = "DevicePlugins"
PluginsInstalled[] = "Heartbeat"
PluginsInstalled[] = "Ecommerce"
PluginsInstalled[] = "UserId"
PluginsInstalled[] = "RssWidget"
PluginsInstalled[] = "Feedback"
PluginsInstalled[] = "Marketplace"
PluginsInstalled[] = "ProfessionalServices"
PluginsInstalled[] = "SecurityInfo"
PluginsInstalled[] = "GeoIp2"
PluginsInstalled[] = "QueuedTracking"
PluginsInstalled[] = "TwoFactorAuth"
PluginsInstalled[] = "DeviceDetectorCache"
PluginsInstalled[] = "LoginLdap"
PluginsInstalled[] = "Login"

[QueuedTracking]
notify_queue_threshold_single_queue = 250000

[LoginLdap]
servers[] = "Miraheze Ldap"
use_ldap_for_authentication = 1
enable_synchronize_access_from_ldap = 0
new_user_default_sites_view_access = "all"
user_email_suffix = ""
required_member_of = ""
required_member_of_field = "memberOf"
ldap_user_filter = "(|(&(memberOf=cn=matomo-super,ou=groups,dc=miraheze,dc=org)(objectclass=person))(&(memberOf=cn=sre,ou=groups,dc=miraheze,dc=org)(objectclass=person))(&(memberOf=cn=matomo-users,ou=groups,dc=miraheze,dc=org)(objectclass=person)))"
ldap_user_id_field = "uid"
ldap_last_name_field = "sn"
ldap_first_name_field = "givenName"
ldap_alias_field = "cn"
ldap_mail_field = "mail"
ldap_password_field = "userPassword"
ldap_view_access_field = "view"
ldap_admin_access_field = "admin"
ldap_superuser_access_field = "superuser"
use_webserver_auth = 0
user_access_attribute_server_specification_delimiter = ";"
user_access_attribute_server_separator = ":"
instance_name = ""
ldap_network_timeout = "15"

[LoginLdap_Miraheze Ldap]
hostname = "ldaps://ldap1.miraheze.org/"
port = 636
base_dn = "dc=miraheze,dc=org"
admin_user = "cn=write-user,dc=miraheze,dc=org"
admin_pass = "<%= <a class='mention' href='https://github.com/ldap_password'>@ldap_password</a> %>"
@paladox commented on December 10th 2020 Contributor

Looking at "Visits in Real-time", it doesn't seem to be showing frequent visits like it used to be, which could be an indication that tracking is broken or something?

"Thursday, December 10, - 13:53:54 (46 min 21s)" is an entry I see.

@paladox commented on December 10th 2020 Contributor

Oh! I'm seeing 204s

"POST https://matomo.miraheze.org/matomo.php?action_name=Blackmailed%20by%20Leighton%20-%20Degrees%20of%20Lewdity%20Wiki&idsite=1&rec=1&r=194995&h=18&m=10&s=8&url=https%3A%2F%2Fdegreesoflewdity.miraheze.org%2Fwiki%2FBlackmailed_by_Leighton&urlref=https%3A%2F%2Fdegreesoflewdity.miraheze.org%2Fwiki%2FLeighton&_id=&_idn=1&_refts=0&send_image=0&cookie=1&res=962x601&pv_id=c5kNmB&pf_net=0&pf_srv=29&pf_tfr=15&pf_dm1=653" 204 0

I didn't see 204s in access log from before we upgraded

@paladox commented on December 10th 2020 Contributor

forcing it use GET isn't fixing it either.

@paladox commented on December 10th 2020 Contributor

Disabling Queued Tracker seems to have got "Visits in Real-time" working again.

@nicwortel commented on December 10th 2020

Do you use a Content Security Policy? Could this be caused by #16935?

@paladox commented on December 10th 2020 Contributor

Yeh, we use a CSP. Though the thing is after disabling Queued tracker, it started tracking again.

@sgiehl commented on December 11th 2020 Member

@juliemoynat the average generation time metric was dropped in Matomo 4. There won't we any new values tracked for it. You can have a look at the Page Performance metrics instead, they should be more valuable.

@paladox commented on December 12th 2020 Contributor

Seems like the visitor count is well too low even though I disabled queued tracking:

Screenshot 2020-12-12 at 15 58 13
@paladox commented on December 12th 2020 Contributor

@sgiehl would you have any idea why? Is it due to improved tracking if you don't allow cookies? E.g we have https://github.com/miraheze/MatomoAnalytics/blob/master/includes/MatomoAnalyticsHooks.php#L73

@tsteur commented on December 14th 2020 Member

Seems like the visitor count is well too low even though I disabled queued tracking:

Do I see this right that it means we can close this issue in queued tracking? https://github.com/matomo-org/plugin-QueuedTracking/issues/143

Maybe it is related to the CSP. Have you tried adjusting it already?

@juliemoynat commented on December 18th 2020

Hi,

I had the same problem of loss of visitors on my site after the installation of Matomo 4.0.3. The installation of the updates 4.0.4 then 4.0.5 did not solve the problem. My site is smaller so the drop was less spectacular. However, the visits indicated in Matomo did not correspond to the curve of the visits of my host nor to that of the Google Search Console.

I had already put a message here that I deleted afterwards. I was saying that the CSP rules update hadn't changed anything. I don't have the "Queued tracking" module installed on my side.

I put a message again because I succeeded in solving my problem but I don't have a clear step to take so that others with the problem can correct it but it might give some clues for Matomo to solve the problem.

  1. First of all, I downloaded locally on my computer all the files of my Matomo in production as well as a dump of the database. I should point out that I don't have a lot of database knowledge - which certainly explains my various problems.
  2. I told myself that I should switch back to Matomo version 3 which was working very well. So I came across this article https://matomo.org/faq/how-to/how-do-i-downgrade-from-matomo-4-to-matomo-3/ that I wanted to follow. However, this article is not dated and doesn't specify the precise versions concerned in a clear way (in the query you can find them - I changed the numbers but I think it was a mistake). I followed the process and it didn't work at all. Matomo gave me an error message and I couldn't open it.
  3. So I put back the files of my initial version of Matomo. Then, I wanted to delete my database to put back my initial dump and the total deletion didn't work (a "foreign key" story that I don't understand). So I reimported my dump over it. I still got error messages (that I didn't understand) about "foreign key" again if my memory is good.
  4. There I went back to Matomo and I had a SQL error displayed in red because a column was not found. I searched on the Internet, I came across a person who said that they had removed all the files from Matomo to put back a clean archive and that it had worked. I tried... But of course, the database was already installed so I had to do the correct installation of the files with the SSH command line on my server. It didn't change anything. The SQL error was still there.
  5. Then, I finally came across this post in a Matomo forum https://forum.matomo.org/t/missing-column-name-after-migrating-from-1-12-to-4-0/39459
    Same SQL error as me. I ran the given SQL command. No more SQL error in Matomo. It seems to be working.
  6. The next day, I thought it would still be better if I uploaded my files from my initial Matomo 4.0.5 that hadn't been installed in command line so that I could have the "File integrity" report in Matomo administration (it's safer for me). So I put every file back.
  7. I also had the idea to install a second Matomo, version 3, and to collect the visits of Matomo 3 and 4 in parallel to prove that there was indeed a problem (see this doc about how to collect analytics into two Matomo: https://developer.matomo.org/guides/tracking-javascript-guide#collect-your-analytics-data-into-two-or-more-piwik-servers). But, I had no idea that, in fact, I had actually succeeded in solving my problem. Since December 13, my visit records have returned to normal.

So, I tinkered with the files and the database and now it's working again. I don't have any explanation for this but something was broken, a priori, in the database of my Matomo.
I hope my story can help and good luckā€¦

@1u commented on December 20th 2020

After updating to 4.0.0, metrics went down (using CSP headers as well)
After updating to 4.0.5 they went up again. Generation time remains 0s.

image
@dns2utf8 check https://github.com/matomo-org/matomo/issues/16935#issuecomment-744097347

@tsteur commented on December 20th 2020 Member

BTW It might be worth enabling tracking logging see https://developer.matomo.org/api-reference/tracking-api#debugging-the-tracker and logging to a file https://matomo.org/faq/troubleshooting/faq_115/

You basically want to make sure in your config/config.ini.php these entries exist:

[log]
log_writers[] = file
log_writers[] = screen

[Tracker]
debug = 1

This will write log data into tmp/logs/matomo.log. Depending on your traffic it might log a LOT of data but it would help observing if any errors appear.

After few hours or so you should remove again the [Tracker]debug=1 setting.

I'm not sure if it was mentioned before but you should in any case check your "Matomo System check" which you find in the Matomo Admin section.

It sounds like maybe some file was not correctly removed when it should have maybe @juliemoynat . That's just a guess though. Maybe this file caused some tracking error or so. I suppose you did check the system report when you noticed the tracking issues?

@tsteur commented on December 20th 2020 Member

BTW It might be worth enabling tracking logging see https://developer.matomo.org/api-reference/tracking-api#debugging-the-tracker and logging to a file https://matomo.org/faq/troubleshooting/faq_115/

You basically want to make sure in your config/config.ini.php these entries exist:

[log]
log_writers[] = file
log_writers[] = screen

[Tracker]
debug = 1

This will write log data into tmp/logs/matomo.log. Depending on your traffic it might log a LOT of data but it would help observing if any errors appear.

After few hours or so you should remove again the [Tracker]debug=1 setting.

I'm not sure if it was mentioned before but you should in any case check your "Matomo System check" which you find in the Matomo Admin section.

It sounds like maybe some file was not correctly removed when it should have maybe @juliemoynat . That's just a guess though. Maybe this file caused some tracking error or so. I suppose you did check the system report when you noticed the tracking issues? (it would have shown if some files needed deleting)

@juliemoynat commented on December 21st 2020

@tsteur I had no error in the "File integrity" section and I still have no error now.
And, I've never seen any error in the "Tracking failures" section.

I've compared my folders: my Matomo 4 folder before I tinkered with files and database and my Matomo 4 after (now).

I only have more files and folders now, not less. Here is my report:

Some new plugins folders but I think it's not important because they are examples and test runner:

  • plugins/ExampleAPI
  • plugins/ExampleCommand
  • plugins/ExampleLogTables
  • plugins/ExamplePlugin
  • plugins/ExampleReport
  • plugins/ExampleSettingsPlugin
  • plugins/ExampleTheme
  • plugins/ExampleTRacker
  • plugins/ExampleUI
  • plugins/ExampleVisualization
  • plugins/TestRunner

I have some empty folders, and some .git and .github folders.
I also have some .gitkeep, .travis.yml, .gitignore, .gitattributes, .coveralls.yml, .bowerrc, .lfsconfig files.
This may be due to the fact I've reinstalled Matomo via SSH command before reuploading my initial Matomo installation on my server. This should not cause a tracking problem I guess.

I have a lot of new files in tmp/templates_c/ and some CSV files in tmp/assets/ but I think it's normal.

And all the files below are in my new installation but were not in my initial installation. I don't know if it's normal or not.

  • tmp/CACHEDIR.TAG
  • tmp/cache/tracker/eagercache-405-tracker.php
  • tmp/cache/tracker/eagercache-405-ui.php
  • tmp/cache/tracker/matomocache_3.php
  • tmp/cache/tracker/matomocache_Archiving.minVisitTime.3.php
  • tmp/cache/tracker/matomocache_bot402-all.php
  • tmp/cache/tracker/matomocache_camera402-all.php
  • tmp/cache/tracker/matomocache_carbrowser402-all.php
  • tmp/cache/tracker/matomocache_console402-all.php
  • tmp/cache/tracker/matomocache_DeviceDetector-402regexes-bot.php
  • tmp/cache/tracker/matomocache_DeviceDetector-402regexes-browser.php
  • tmp/cache/tracker/matomocache_DeviceDetector-402regexes-browserengine.php
  • tmp/cache/tracker/matomocache_DeviceDetector-402regexes-camera.php
  • tmp/cache/tracker/matomocache_DeviceDetector-402regexes-carbrowser.php
  • tmp/cache/tracker/matomocache_DeviceDetector-402regexes-console.php
  • tmp/cache/tracker/matomocache_DeviceDetector-402regexes-feedreader.php
  • tmp/cache/tracker/matomocache_DeviceDetector-402regexes-mediaplayer.php
  • tmp/cache/tracker/matomocache_DeviceDetector-402regexes-mobile.php
  • tmp/cache/tracker/matomocache_DeviceDetector-402regexes-mobileapp.php
  • tmp/cache/tracker/matomocache_DeviceDetector-402regexes-pim.php
  • tmp/cache/tracker/matomocache_DeviceDetector-402regexes-portablemediaplayer.php
  • tmp/cache/tracker/matomocache_DeviceDetector-402regexes-vendorfragments.php
  • tmp/cache/tracker/matomocache_feedreader402-all.php
  • tmp/cache/tracker/matomocache_mediaplayer402-all.php
  • tmp/cache/tracker/matomocache_mobileapp402-all.php
  • tmp/cache/tracker/matomocache_pim402-all.php
  • tmp/cache/tracker/matomocache_portablemediaplayer402-all.php
  • tmp/cache/tracker/matomocache_Translations-en-0f3c54a1b760aca91724465939152677d70c9016.php
  • tmp/cache/tracker/matomocache_Translations-en-e8ab5d02eea827a8cc2a31894c9a36f115db30c4.php
  • tmp/cache/tracker/matomocache_Translations-fr-e8ab5d02eea827a8cc2a31894c9a36f115db30c4.php

That's all.

Powered by GitHub Issue Mirror